diff --git a/.bazelrc b/.bazelrc
deleted file mode 100644
index 785a5d6..0000000
--- a/.bazelrc
+++ /dev/null
@@ -1,12 +0,0 @@
-# Import TensorFlow configuration.
-import %workspace%/tensorflow/.tf_configure.bazelrc
-
-# Coloring for error messages.
-common --color=yes
-
-# Always print test errors.
-test --test_output=errors
-
-# Other build flags.
-build --define=grpc_no_ares=true
-test --define=grpc_no_ares=true
diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index c37242c..0000000
--- a/.gitignore
+++ /dev/null
@@ -1,12 +0,0 @@
-.DS_Store
-.ipynb_checkpoints
-node_modules
-/.bazelrc
-/bazel-*
-/bazel_pip
-/pip_test
-/_python_build
-*.pyc
-__pycache__
-*.swp
-.vscode/
diff --git a/BUILD b/BUILD
deleted file mode 100644
index b5e7ca5..0000000
--- a/BUILD
+++ /dev/null
@@ -1,23 +0,0 @@
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# =============================================================================
-sh_binary(
- name = "pip_pkg",
- srcs = ["pip_pkg.sh"],
- data = [
- "MANIFEST.in",
- "setup.py",
- "//tensorflow_lattice",
- ],
-)
diff --git a/INSTALL.md b/INSTALL.md
deleted file mode 100644
index f8c7cb2..0000000
--- a/INSTALL.md
+++ /dev/null
@@ -1,202 +0,0 @@
-
-# TensorFlow Lattice installation
-
-TensorFlow Lattice runs on Ubuntu and Mac OS X, and requires TensorFlow.
-
-We highly recommend to read [TensorFlow installation
-instructions](https://www.tensorflow.org/install), especially [Installing
-TensorFlow on Ubuntu](https://www.tensorflow.org/install/install_linux) to
-understand virtualenv and pip, and [Installing TensorFlow from
-Sources](https://www.tensorflow.org/install/install_sources).
-
-# Install the prebuilt pip package
-
-## Activate virtualenv
-If using virtualenv, activate your virtualenv for the rest of the installation,
-otherwise skip this step:
-
-``` shell
-~$ virtualenv --system-site-packages tensorflow-lattice # for Python 2.7
-~$ virtualenv --system-site-packages -p python3 tensorflow-lattice # for Python 3.n
-```
-
-Here you can change `tensorflow-lattice` to another target directory you want to
-use.
-
-```shell
-~$ source tensorflow-lattice/bin/activate # bash, sh, ksh, or zsh
-~$ source tensorflow-lattice/bin/activate.csh # csh or tcsh
-```
-
-## Install pip packages.
-You can use pip install to install tensorflow-lattice pip package.
-
-```shell
-(tensorflow-lattice)$ pip install --upgrade tensorflow-lattice # for Python 2.7
-(tensorflow-lattice)$ pip3 install --upgrade tensorflow-lattice # for Python 3.n
-(tensorflow-lattice)$ pip install --upgrade tensorflow-lattice-gpu # for Python 2.7 and GPU
-(tensorflow-lattice)$ pip3 install --upgrade tensorflow-lattice-gpu # for Python 3.n and GPU
-```
-Our custom operators do not have GPU kernels. The main difference
-between `tensorflow-lattice-gpu` and `tensorflow-lattice` pip package is that
-the former requires `tensorflow-gpu` pip package whereas the latter requires
-`tensorflow` pip package.
-
-## Test TensorFlow and TensorFlow Lattice
-
-Run the following python script to test TensorFlow Lattice.
-
-```python
-import tensorflow as tf
-import tensorflow_lattice as tfl
-
-x = tf.compat.v1.placeholder(tf.float32, shape=(None, 2))
-(y, _, _, _) = tfl.lattice_layer(x, lattice_sizes=(2, 2))
-
-with tf.Session() as sess:
- sess.run(tf.global_variables_initializer())
- print(sess.run(y, feed_dict={x: [[0.0, 0.0]]}))
-```
-
-Now you are ready to use *TensorFlow Lattice*. Check out examples in the
-[examples](https://github.com/tensorflow/lattice/tree/master/examples) directory
-and run them if you need more examples to run.
-[Tutorial](g3doc/tutorial/index.md) contains detailed explanation on how to use
-TensorFlow Lattice.
-
-You can stop here unless you want to build TensorFlow Lattice from the source.
-
-# Build TensorFlow Lattice and TensorFlow pip package from the source.
-You can also build TensorFlow Lattice packages from the source.
-For this, you will need to compile all libraries using
-[Bazel](https://bazel.build) against TensorFlow headers.
-
-
-We will show how to build TensorFlow and TensorFlow Lattice pip package using
-Bazel, and install it to your virtualenv.
-
-## Activate virtualenv
-
-If using virtualenv, activate your virtualenv for the rest of the installation,
-otherwise skip this step:
-
-```shell
-~$ source $VIRTUALENV_PATH/bin/activate # bash, sh, ksh, or zsh
-~$ source $VIRTUALENV_PATH/bin/activate.csh # csh or tcsh
-```
-
-or if you are using virtualenv for the first time,
-
-```shell
-~$ sudo apt-get install python-virtualenv
-~$ virtualenv --system-site-packages tensorflow-lattice
-~$ source ~/tensorflow-lattice/bin/activate # bash, sh, ksh, or zsh
-~$ source ~/tensorflow-lattice/bin/activate.csh # csh or tcsh
-```
-## Prepare TensorFlow envirnoment for Linux.
-
-Please follow instructions in [Prepare environment for
-Linux](https://www.tensorflow.org/install/install_sources#prepare_environment_for_linux)
-to setup the environment for TensorFlow.
-
-## Clone the TensorFlow Lattice repository.
-
-Let us clone the TensorFlow Lattice repository, which contains TensorFlow as a
-submodule:
-
-```shell
-(tensorflow-lattice)~$ git clone --recursive https://github.com/tensorflow/lattice.git
-```
-
-## Configure TensorFlow and build TensorFlow pip package.
-
-### Configure TensorFlow
-
-We now need to configure TensorFlow options. See [Configure the
-installation](https://www.tensorflow.org/install/install_sources#configure_the_installation)
-for the details.
-
-```shell
-(tensorflow-lattice)~$ cd lattice
-(tensorflow-lattice)~/lattice$ cd tensorflow
-(tensorflow-lattice)~/lattice/tensorflow$ ./configure
-```
-
-### Build TensorFlow pip packaging script
-
-We are ready to build the TensorFlow pip package. See [Build the pip
-package](https://www.tensorflow.org/install/install_sources#build_the_pip_package)
-for the details.
-
-To build a pip package for TensorFlow with CPU-only support:
-
-```shell
-(tensorflow-lattice)~/lattice/tensorflow$ bazel build \
- --config=opt \
- tensorflow/tools/pip_package:build_pip_package
-```
-
-To build a pip package for TensorFlow with GPU support:
-
-```shell
-(tensorflow-lattice)~/lattice/tensorflow$ bazel build \
- --config=cuda \
- tensorflow/tools/pip_package:build_pip_package
-```
-
-### Install TensorFlow pip package
-
-```shell
-(tensorflow-lattice)~/lattice/tensorflow$ bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg
-(tensorflow-lattice)~/lattice/tensorflow$ pip install /tmp/tensorflow_pkg/*.whl
-```
-
-### Build TensorFlow Lattice pip packaging script
-
-To build a pip package for TensorFlow with CPU-only support:
-
-```shell
-(tensorflow-lattice)~/$ cd ~/lattice
-(tensorflow-lattice)~/lattice$ bazel build \
- --config=opt :pip_pkg
-```
-
-### Install TensorFlow Lattice pip package
-
-```shell
-(tensorflow-lattice)~/lattice$ bazel-bin/pip_pkg /tmp/tensorflow_lattice_pkg
-(tensorflow-lattice)~/lattice$ pip install /tmp/tensorflow_lattice_pkg/*.whl
-```
-
-### Test TensorFlow and TensorFlow Lattice
-```shell
-(tensorflow-lattice)~/lattice$ cd examples
-(tensorflow-lattice)~/lattice/examples$ python test.py
-```
-
-test.py is a simple python script.
-
-```python
-import tensorflow as tf
-import tensorflow_lattice as tfl
-
-x = tf.compat.v1.placeholder(tf.float32, shape=(None, 2))
-(y, _, _, _) = tfl.lattice_layer(x, lattice_sizes=(2, 2))
-
-with tf.Session() as sess:
- sess.run(tf.global_variables_initializer())
- print(sess.run(y, feed_dict={x: [[0.0, 0.0]]}))
-```
diff --git a/MANIFEST.in b/MANIFEST.in
deleted file mode 100644
index 3b46ccd..0000000
--- a/MANIFEST.in
+++ /dev/null
@@ -1,6 +0,0 @@
-include README.md LICENSE BUILD
-recursive-include tensorflow_lattice BUILD
-graft tensorflow_lattice/cc
-recursive-exclude tensorflow_lattice/cc *_test.cc *.so
-recursive-exclude tensorflow_lattice/cc/test_tools *
-
diff --git a/README.md b/README.md
index 230915f..71e848e 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-
-
-
-
-
# TensorFlow Lattice
-This is an implementation of [Monotonic Calibrated Interpolated Look-Up Tables](http://jmlr.org/papers/v17/15-243.html) in [TensorFlow](https://www.tensorflow.org).
+TensorFlow Lattice is a library that implements constrained and interpretable
+lattice based models. It is an implementation of
+[Monotonic Calibrated Interpolated Look-Up Tables](http://jmlr.org/papers/v17/15-243.html)
+in [TensorFlow](https://www.tensorflow.org).
-These are fast-to-evaluate and interpretable lattice models, also known as
-interpolated look-up tables. This library also provides a rich and intuitive set
-of regularizations and monotonicity constraints configurable per feature.
+The library enables you to inject domain knowledge into
+the learning process through common-sense or policy-driven shape constraints.
+This is done using a collection of Keras layers that can satisfy constraints
+such as monotonicity, convexity and pairwise trust:
-It includes
-[__TensorFlow estimators__](https://www.tensorflow.org/extend/estimators) for
-regression and classification with the most common set ups for lattice models:
+* PWLCalibration: piecewise linear calibration of signals.
+* CategoricalCalibration: mapping of categorical inputs into real values.
+* Lattice: interpolated look-up table implementation.
+* Linear: linear function with monotonicity and norm constraints.
+
+The library also provides easy to setup canned estimators for common use cases:
* Calibrated Linear
* Calibrated Lattice
-* Random Tiny Lattices (_RTL_)
-* Embedded Tiny Lattices (_ETL_) (see [Deep Lattice Networks and Partial Monotonic Functions](https://research.google.com/pubs/pub46327.html))
-
-Additionally this library provides two types of __model components__
-(or __layers__) that can be combined with other types of models (including
-neural networks):
+* Random Tiny Lattices (RTL)
+* Crystals
-* Calibration: piecewise linear calibration of signals.
-* Lattice: interpolated look-up table implementation.
+With TF Lattice you can use domain knowledge to better extrapolate to the parts
+of the input space not covered by the training dataset. This helps avoid
+unexpected model behaviour when the serving distribution is different from the
+training distribution.
+
+
+
You can install our prebuilt pip package using
```bash
pip install tensorflow-lattice
```
-
-but please see the [install](INSTALL.md) section for more detailed instructions.
-
-This [tutorial](g3doc/tutorial/index.md) contains more detailed explanation
-about lattice models and usage in TensorFlow, and check out
-[API docs](g3doc/api_docs/python/index.md) for python APIs.
-
-__TensorFlow Lattice is not an official Google product.__
diff --git a/WORKSPACE b/WORKSPACE
index 0f4283f..06761c4 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1,77 +1,16 @@
-# Copyright 2017 The TensorFlow Lattice Authors.
+# Copyright 2018 The TensorFlow Lattice Authors.
#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
# ==============================================================================
-workspace(name = "tensorflow_lattice")
-
-local_repository(
- name = "org_tensorflow",
- path = "tensorflow",
-)
-
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
-
-# This rule is from TensorFlow's WORKSPACE.
-http_archive(
- name = "io_bazel_rules_closure",
- sha256 = "e0a111000aeed2051f29fcc7a3f83be3ad8c6c93c186e64beb1ad313f0c7f9f9",
- strip_prefix = "rules_closure-cf1e44edb908e9616030cc83d085989b8e6cd6df",
- urls = [
- "http://mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz",
- "https://github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz", # 2019-04-04
- ],
-)
-# Apple and Swift rules.
-http_archive(
- name = "build_bazel_rules_apple",
- sha256 = "23792cd999f97fc97284d1c44cb1324bfdd0bc54aa68ad513fa3705aca3b1f9e",
- urls = ["https://github.com/bazelbuild/rules_apple/releases/download/0.15.0/rules_apple.0.15.0.tar.gz"],
-) # https://github.com/bazelbuild/rules_apple/releases
-http_archive(
- name = "build_bazel_apple_support",
- sha256 = "7356dbd44dea71570a929d1d4731e870622151a5f27164d966dda97305f33471",
- urls = ["https://github.com/bazelbuild/apple_support/releases/download/0.6.0/apple_support.0.6.0.tar.gz"],
-) # https://github.com/bazelbuild/apple_support/releases
-http_archive(
- name = "bazel_skylib",
- sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
- urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/0.8.0/bazel-skylib.0.8.0.tar.gz"],
-) # https://github.com/bazelbuild/bazel-skylib/releases
-http_archive(
- name = "build_bazel_rules_swift",
- sha256 = "9efe9699e9765e6b4a5e063e4a08f6b163cccaf0443f775d935baf5c3cd6ed0e",
- urls = ["https://github.com/bazelbuild/rules_swift/releases/download/0.9.0/rules_swift.0.9.0.tar.gz"],
-) # https://github.com/bazelbuild/rules_swift/releases
-http_archive(
- name = "com_github_apple_swift_swift_protobuf",
- type = "zip",
- strip_prefix = "swift-protobuf-1.5.0/",
- urls = ["https://github.com/apple/swift-protobuf/archive/1.5.0.zip"],
-) # https://github.com/apple/swift-protobuf/releases
-http_file(
- name = "xctestrunner",
- executable = 1,
- urls = ["https://github.com/google/xctestrunner/releases/download/0.2.7/ios_test_runner.par"],
-) # https://github.com/google/xctestrunner/releases
-# Use `swift_rules_dependencies` to fetch the toolchains. With the
-# `git_repository` rules above, the following call will skip redefining them.
-load("@build_bazel_rules_swift//swift:repositories.bzl", "swift_rules_dependencies")
-swift_rules_dependencies()
-
-load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
-
-tf_workspace(
- path_prefix = "",
- tf_repo_name = "org_tensorflow",
-)
+workspace(name = "tensorflow_lattice")
diff --git a/build_docs.py b/build_docs.py
new file mode 100644
index 0000000..74c317b
--- /dev/null
+++ b/build_docs.py
@@ -0,0 +1,85 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Generate docs API for TF Lattice.
+
+Example run:
+
+```
+python build_docs.py --output_dir=/path/to/output
+```
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import sys
+
+from absl import app
+from absl import flags
+
+from tensorflow_docs.api_generator import generate_lib
+from tensorflow_docs.api_generator import public_api
+
+import tensorflow_lattice as tfl
+
+flags.DEFINE_string('output_dir', '/tmp/tfl_api/',
+ 'The path to output the files to')
+
+flags.DEFINE_string(
+ 'code_url_prefix',
+ 'https://github.com/tensorflow/lattice/blob/master/tensorflow_lattice',
+ 'The url prefix for links to code.')
+
+flags.DEFINE_bool('search_hints', True,
+ 'Include metadata search hints in the generated files')
+
+flags.DEFINE_string('site_path', 'lattice/api_docs/python',
+ 'Path prefix in the _toc.yaml')
+
+FLAGS = flags.FLAGS
+
+
+def local_definitions_filter(path, parent, children):
+ """Filters local imports, except for the tfl.layers module."""
+ if path == ('tfl', 'layers'):
+ return children
+ return public_api.local_definitions_filter(path, parent, children)
+
+
+def main(_):
+ private_map = {
+ 'tfl': ['python'],
+ 'tfl.categorical_calibration_layer': ['CategoricalCalibration'],
+ 'tfl.lattice_layer': ['Lattice'],
+ 'tfl.linear_layer': ['Linear'],
+ 'tfl.pwl_calibration_layer': ['PWLCalibration'],
+ 'tfl.parallel_combination_layer': ['ParallelCombination']
+ }
+ doc_generator = generate_lib.DocGenerator(
+ root_title='TensorFlow Lattice 2.0',
+ py_modules=[('tfl', tfl)],
+ base_dir=os.path.dirname(tfl.__file__),
+ code_url_prefix=FLAGS.code_url_prefix,
+ search_hints=FLAGS.search_hints,
+ site_path=FLAGS.site_path,
+ private_map=private_map,
+ callbacks=[local_definitions_filter])
+
+ sys.exit(doc_generator.build(output_dir=FLAGS.output_dir))
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/build_tools/ci_build/ci_common.sh b/build_tools/ci_build/ci_common.sh
deleted file mode 100644
index d368c68..0000000
--- a/build_tools/ci_build/ci_common.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# Run tensorflow lattice bazel tests.
-function tensorflow_lattice_test {
- # Cleaning up bazel workspace
- bazel clean
-
- if [[ "${IS_MAC}" == true ]]; then
- N_JOBS=$(sysctl -n hw.ncpu)
- else
- N_JOBS=$(grep -c ^processor /proc/cpuinfo)
- fi
-
- echo ""
- echo "Bazel will use ${N_JOBS} concurrent job(s)."
- echo ""
-
- bazel test --config=opt --test_tag_filters=-gpu -k \
- --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --build_tests_only \
- --test_output=errors -- \
- //tensorflow_lattice/...
-}
diff --git a/build_tools/ci_build/macosx/py2.sh b/build_tools/ci_build/macosx/py2.sh
deleted file mode 100755
index b36846b..0000000
--- a/build_tools/ci_build/macosx/py2.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# This script will run the bash function tensorflow_lattice_test under a python2
-# environment.
-
-set -e
-set -x
-
-# Source common scripts.
-source "build_tools/common.sh"
-
-export IS_MAC=true
-export TFL_PY="py2"
-export TFL_USE_GPU=false
-
-# Prepare build.
-prepare_build
-
-# Source common ci scripts.
-source "build_tools/ci_build/ci_common.sh"
-
-# Activate virtualenv.
-source ${TFL_ENV_PATH}/bin/activate
-
-echo "Running all tests."
-tensorflow_lattice_test
-echo "Done with testing."
-
-deactivate
diff --git a/build_tools/ci_build/macosx/py3.sh b/build_tools/ci_build/macosx/py3.sh
deleted file mode 100755
index 14bccc8..0000000
--- a/build_tools/ci_build/macosx/py3.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# This script will run the bash function tensorflow_lattice_test under a python3
-# environment.
-
-set -e
-set -x
-
-# Source common scripts.
-source "build_tools/common.sh"
-
-export IS_MAC=true
-export TFL_PY="py3"
-export TFL_USE_GPU=false
-
-# Prepare build.
-prepare_build
-
-# Source common ci scripts.
-source "build_tools/ci_build/ci_common.sh"
-
-# Activate virtualenv.
-source ${TFL_ENV_PATH}/bin/activate
-
-echo "Running all tests."
-tensorflow_lattice_test
-echo "Done with testing."
-
-deactivate
diff --git a/build_tools/ci_build/ubuntu/py2.sh b/build_tools/ci_build/ubuntu/py2.sh
deleted file mode 100755
index bd014b1..0000000
--- a/build_tools/ci_build/ubuntu/py2.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# This script will run the bash function tensorflow_lattice_test under a python2
-# environment.
-
-set -e
-set -x
-
-# Source common scripts.
-source "build_tools/common.sh"
-
-export IS_MAC=false
-export TFL_PY="py2"
-export TFL_USE_GPU=false
-
-# Prepare build.
-prepare_build
-
-# Source common ci scripts.
-source "build_tools/ci_build/ci_common.sh"
-
-# Activate virtualenv.
-source ${TFL_ENV_PATH}/bin/activate
-
-echo "Running all tests."
-tensorflow_lattice_test
-echo "Done with testing."
-
-deactivate
diff --git a/build_tools/ci_build/ubuntu/py3.sh b/build_tools/ci_build/ubuntu/py3.sh
deleted file mode 100755
index 9e223ef..0000000
--- a/build_tools/ci_build/ubuntu/py3.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# This script will run the bash function tensorflow_lattice_test under a python3
-# environment.
-
-set -e
-set -x
-
-# Source common scripts.
-source "build_tools/common.sh"
-
-export IS_MAC=false
-export TFL_PY="py3"
-export TFL_USE_GPU=false
-
-# Prepare build.
-prepare_build
-
-# Source common ci scripts.
-source "build_tools/ci_build/ci_common.sh"
-
-# Activate virtualenv.
-source ${TFL_ENV_PATH}/bin/activate
-
-echo "Running all tests."
-tensorflow_lattice_test
-echo "Done with testing."
-
-deactivate
diff --git a/build_tools/common.sh b/build_tools/common.sh
deleted file mode 100755
index e512180..0000000
--- a/build_tools/common.sh
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# Git initialization
-function git_init {
- # Run configure.
- export TF_NEED_GCP=0
- export TF_NEED_HDFS=0
- export PYTHON_BIN_PATH=$(which python)
-
- # Initialize git.
- git init
-
- if [ -d "tensorflow" ]; then
- echo "TensorFlow submodule exist. Checkout r1.14"
- cd tensorflow
- git checkout r1.14
- cd -
- else
- echo "Add TensorFlow r1.14 submodule."
- git submodule add -b r1.14 https://github.com/tensorflow/tensorflow.git
- fi
-
- # Fetch all submodules.
- git submodule update --init --recursive
-
- # Configure tensorflow.
- cd tensorflow
- git show --oneline -s
- yes "" | ./configure
-
- cd -
- echo "Applying visibility change."
- sed -i.bak -e 's/:internal/\/\/visibility:public/g' -- "tensorflow/tensorflow/BUILD"
-}
-
-# Create virtualenv.
-function create_virtualenv {
- if [ "${TFL_PY}" = "py3" ]; then
- echo "Setting up python 3 virtualenv"
- export TFL_ENV_PATH=${TFL_ROOT}/tensorflow-lattice-env-py3
- virtualenv --system-site-packages -p python3 ${TFL_ENV_PATH}
- else
- echo "Setting up python 2 virtualenv"
- export TFL_ENV_PATH=${TFL_ROOT}/tensorflow-lattice-env-py2
- virtualenv --system-site-packages -p python2.7 ${TFL_ENV_PATH}
- fi
- source ${TFL_ENV_PATH}/bin/activate
- python -V
- pip install --upgrade pip
- pip install --upgrade six numpy wheel enum34 protobuf keras_applications keras_preprocessing tensorflow_estimator
- deactivate
-}
-
-# Prepare all necessary environment for bazel build & testing.
-function prepare_build {
- # modify default gcc on linux
- if [ "$(uname)" == "Linux" ]; then
- sudo update-alternatives --set gcc /usr/bin/gcc-4.8
- fi
-
- # If TFL_ROOT does not exist, create one in here.
- if [ -z "${TFL_ROOT}" ]; then
- echo "TFL_ROOT is empty, so set to /tmp/tfl_root."
- export TFL_ROOT="/tmp/tfl_root"
- fi
-
- # Create virtualenv.
- create_virtualenv
-
- # Activate virtualenv.
- source ${TFL_ENV_PATH}/bin/activate
-
- if [ "${TFL_USE_GPU}" = true ]; then
- echo "GPU build -- Enable CUDA"
- export TF_NEED_CUDA=1
- else
- echo "CPU build -- No CUDA"
- export TF_NEED_CUDA=0
- fi
-
- echo "Initialize git repo."
- git_init
- echo "Initialization is done."
-
- deactivate
-}
diff --git a/build_tools/release_build/py2.sh b/build_tools/release_build/py2.sh
deleted file mode 100755
index 74de1dc..0000000
--- a/build_tools/release_build/py2.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-set -e
-set -x
-
-# Source common scripts.
-source "build_tools/common.sh"
-
-export TFL_PY="py2"
-export TFL_USE_GPU=false
-
-# Prepare build.
-prepare_build
-
-# Source common release scripts.
-source "build_tools/release_build/release_common.sh"
-
-# Activate virtualenv.
-source ${TFL_ENV_PATH}/bin/activate
-
-echo "Build pip package."
-build_pip_pkg
-echo "Done."
-
-echo "Install pip package and test."
-install_pip_and_test
-echo "Done."
-
-deactivate
diff --git a/build_tools/release_build/py2_gpu.sh b/build_tools/release_build/py2_gpu.sh
deleted file mode 100755
index 2731300..0000000
--- a/build_tools/release_build/py2_gpu.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-set -e
-set -x
-
-# Source common scripts.
-source "build_tools/common.sh"
-
-export TFL_PY="py2"
-export TFL_USE_GPU=true
-
-# Prepare build.
-prepare_build
-
-# Source common release scripts.
-source "build_tools/release_build/release_common.sh"
-
-# Activate virtualenv.
-source ${TFL_ENV_PATH}/bin/activate
-
-echo "Build pip package."
-build_pip_pkg
-echo "Done."
-
-echo "Install pip package and test."
-install_pip_and_test
-echo "Done."
-
-deactivate
diff --git a/build_tools/release_build/py3.sh b/build_tools/release_build/py3.sh
deleted file mode 100755
index 89a2f04..0000000
--- a/build_tools/release_build/py3.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-set -e
-set -x
-
-# Source common scripts.
-source "build_tools/common.sh"
-
-export TFL_PY="py3"
-export TFL_USE_GPU=false
-
-# Prepare build.
-prepare_build
-
-# Source common release scripts.
-source "build_tools/release_build/release_common.sh"
-
-# Activate virtualenv.
-source ${TFL_ENV_PATH}/bin/activate
-
-echo "Build pip package."
-build_pip_pkg
-echo "Done."
-
-echo "Install pip package and test."
-install_pip_and_test
-echo "Done."
-
-deactivate
diff --git a/build_tools/release_build/py3_gpu.sh b/build_tools/release_build/py3_gpu.sh
deleted file mode 100755
index a84defc..0000000
--- a/build_tools/release_build/py3_gpu.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-set -e
-set -x
-
-# Source common scripts.
-source "build_tools/common.sh"
-
-export TFL_PY="py3"
-export TFL_USE_GPU=true
-
-# Prepare build.
-prepare_build
-
-# Source common release scripts.
-source "build_tools/release_build/release_common.sh"
-
-# Activate virtualenv.
-source ${TFL_ENV_PATH}/bin/activate
-
-echo "Build pip package."
-build_pip_pkg
-echo "Done."
-
-echo "Install pip package and test."
-install_pip_and_test
-echo "Done."
-
-deactivate
diff --git a/build_tools/release_build/release_common.sh b/build_tools/release_build/release_common.sh
deleted file mode 100644
index c27b783..0000000
--- a/build_tools/release_build/release_common.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-function build_pip_pkg {
- # Clean up bazel workspace
- bazel clean
-
- if [ "${TFL_NATIVE}" = true ]; then
- # Build pip install package.
- bazel build \
- --define framework_shared_object=true \
- --copt="-D_GLIBCXX_USE_CXX11_ABI=0" \
- --compilation_mode=opt \
- --distinct_host_configuration=false \
- :pip_pkg
- else
- bazel build \
- --define framework_shared_object=true \
- --copt="-D_GLIBCXX_USE_CXX11_ABI=0" \
- --compilation_mode=opt \
- --cpu=k8 \
- --distinct_host_configuration=false \
- :pip_pkg
- fi
-
- if [ -z "${TFL_ARTIFACTS_DIR}" ]; then
- echo "TFL_ARTIFACTS_DIR is empty, so set tp /tmp/tfl_artifacts"
- export TFL_ARTIFACTS_DIR="/tmp/tfl_artifacts"
- fi
-
- # Create wheel to artifacts dir.
- if [ "${TFL_USE_GPU}" = true ]; then
- echo 'Building pip package for gpu'
- ./bazel-bin/pip_pkg ${TFL_ARTIFACTS_DIR} --gpu
- else
- echo 'Building pip package for cpu'
- ./bazel-bin/pip_pkg ${TFL_ARTIFACTS_DIR}
- fi
-}
-
-function install_pip_and_test {
- # Check python version.
- python -V
-
- # Install TensorFlow Lattice
- pip install --upgrade ${TFL_ARTIFACTS_DIR}/*.whl
-
- # Run the example script to check whether it works or not.
- cd examples
-
- # Check TensorFlow version
- python -c 'import tensorflow as tf; print(tf.__version__)'
-
- echo 'running lattice example'
- python lattice_test.py
- echo 'running coffee example'
- python coffee_test.py
- echo 'running estimator example'
- python estimator_test.py
-}
diff --git a/docs/_book.yaml b/docs/_book.yaml
new file mode 100644
index 0000000..09c62f4
--- /dev/null
+++ b/docs/_book.yaml
@@ -0,0 +1,35 @@
+upper_tabs:
+# Tabs left of dropdown menu
+- include: /_upper_tabs_left.yaml
+- include: /api_docs/_upper_tabs_api.yaml
+# Dropdown menu
+- name: Resources
+ path: /resources
+ is_default: true
+ menu:
+ - include: /resources/_menu_toc.yaml
+ lower_tabs:
+ # Subsite tabs
+ other:
+ - name: Guide & Tutorials
+ contents:
+ - title: Overview
+ path: /lattice/overview
+ - title: Install
+ path: /lattice/install
+ - heading: TensorFlow Lattice Tutorials
+ - title: Shape Constraints Tutorial
+ path: /lattice/tutorials/shape_constraints
+ - title: Keras Layers Tutorial
+ path: /lattice/tutorials/keras_layers
+ - title: Canned Estmators Tutorial
+ path: /lattice/tutorials/canned_estimators
+ - title: Custom Estimators Tutorial
+ path: /lattice/tutorials/custom_estimators
+
+ - name: API
+ skip_translation: true
+ contents:
+ - include: /lattice/api_docs/python/_toc.yaml
+
+- include: /_upper_tabs_right.yaml
diff --git a/docs/_index.yaml b/docs/_index.yaml
new file mode 100644
index 0000000..e2ff8e9
--- /dev/null
+++ b/docs/_index.yaml
@@ -0,0 +1,81 @@
+book_path: /lattice/_book.yaml
+project_path: /lattice/_project.yaml
+description:
+landing_page:
+ custom_css_path: /site-assets/css/style.css
+ rows:
+ - heading: Flexible, controlled and interpretable ML with lattice based models
+ items:
+ - classname: devsite-landing-row-50
+ description: >
+
TensorFlow Lattice is a library that implements constrained and interpretable lattice
+ based models. The library enables you to inject domain knowledge into the learning process
+ through common-sense or policy-driven
+ shape constraints. This is done using a
+ collection of Keras layers that can satisfy
+ constraints such as monotonicity, convexity and how features interact. The library also
+ provides easy to setup canned estimators.
+
With TF Lattice you can use domain knowledge to better extrapolate to the parts of the
+ input space not covered by the training dataset. This helps avoid unexpected model behaviour
+ when the serving distribution is different from the training distribution.
+
+ - classname: devsite-landing-row-cards
+ items:
+ - heading: "TensorFlow Lattice: Flexibility empowered by prior knowledge"
+ image_path: /resources/images/tf-logo-card-16x9.png
+ path: https://ai.googleblog.com/2017/10/tensorflow-lattice-flexibility.html
+ buttons:
+ - label: "Read on the Google AI blog"
+ path: https://ai.googleblog.com/2017/10/tensorflow-lattice-flexibility.html
+ - heading: "TensorFlow Lattice: Control your ML with monotonicity"
+ youtube_id: ABBnNjbjv2Q
+ buttons:
+ - label: Watch the video
+ path: https://www.youtube.com/watch?v=ABBnNjbjv2Q
+ - heading: "TF Lattice on GitHub"
+ image_path: /resources/images/github-card-16x9.png
+ path: https://github.com/tensorflow/lattice
+ buttons:
+ - label: "View on GitHub"
+ path: https://github.com/tensorflow/lattice
diff --git a/g3doc/images/2d_lattice.png b/docs/images/2d_lattice.png
similarity index 100%
rename from g3doc/images/2d_lattice.png
rename to docs/images/2d_lattice.png
diff --git a/docs/images/data_dist.png b/docs/images/data_dist.png
new file mode 100644
index 0000000..63eceab
Binary files /dev/null and b/docs/images/data_dist.png differ
diff --git a/docs/images/favicon.ico b/docs/images/favicon.ico
new file mode 100644
index 0000000..41c37e4
Binary files /dev/null and b/docs/images/favicon.ico differ
diff --git a/docs/images/flexible_fit.png b/docs/images/flexible_fit.png
new file mode 100644
index 0000000..1957eb5
Binary files /dev/null and b/docs/images/flexible_fit.png differ
diff --git a/docs/images/linear_fit.png b/docs/images/linear_fit.png
new file mode 100644
index 0000000..ad032f0
Binary files /dev/null and b/docs/images/linear_fit.png differ
diff --git a/docs/images/model_comparison.png b/docs/images/model_comparison.png
new file mode 100644
index 0000000..5213b4b
Binary files /dev/null and b/docs/images/model_comparison.png differ
diff --git a/docs/images/monotonic_fit.png b/docs/images/monotonic_fit.png
new file mode 100644
index 0000000..8e62a76
Binary files /dev/null and b/docs/images/monotonic_fit.png differ
diff --git a/docs/images/pwl_calibration_distance.png b/docs/images/pwl_calibration_distance.png
new file mode 100644
index 0000000..63108f2
Binary files /dev/null and b/docs/images/pwl_calibration_distance.png differ
diff --git a/docs/images/pwl_calibration_price.png b/docs/images/pwl_calibration_price.png
new file mode 100644
index 0000000..1de26c2
Binary files /dev/null and b/docs/images/pwl_calibration_price.png differ
diff --git a/docs/images/regularized_fit.png b/docs/images/regularized_fit.png
new file mode 100644
index 0000000..96f4355
Binary files /dev/null and b/docs/images/regularized_fit.png differ
diff --git a/g3doc/images/tensorflow_lattice.png b/docs/images/tensorflow_lattice.png
similarity index 100%
rename from g3doc/images/tensorflow_lattice.png
rename to docs/images/tensorflow_lattice.png
diff --git a/docs/install.md b/docs/install.md
new file mode 100644
index 0000000..9937349
--- /dev/null
+++ b/docs/install.md
@@ -0,0 +1,39 @@
+# Install TensorFlow Lattice
+
+There are several ways to set up your environment to use TensorFlow Lattice
+(TFL).
+
+* The easiest way to learn and use TFL requires no installation: run the any
+ of the tutorials (e.g.
+ [canned estimators tutorial](tutorials/canned_estimators.ipynb)).
+* To use TFL on a local machine, install the `tensorflow-lattice` pip package.
+* If you have a unique machine configuration, you can build the package from
+ source.
+
+## Install TensorFlow Lattice using pip
+
+Install using pip.
+
+```shell
+pip install --upgrade tensorflow-lattice
+```
+
+## Build from source
+
+Clone the github repo:
+
+```shell
+git clone https://github.com/tensorflow/lattice.git
+```
+
+Build pip package from source:
+
+```shell
+python setup.py sdist bdist_wheel --universal --release
+```
+
+Install the package:
+
+```shell
+pip install --user --upgrade /path/to/pkg.whl
+```
diff --git a/docs/overview.md b/docs/overview.md
new file mode 100644
index 0000000..ba1d9e0
--- /dev/null
+++ b/docs/overview.md
@@ -0,0 +1,200 @@
+# TensorFlow Lattice (TFL)
+
+TensorFlow Lattice is a library that implements flexible, controlled and
+interpretable lattice based models. The library enables you to inject domain
+knowledge into the learning process through common-sense or policy-driven
+[shape constraints](tutorials/shape_constraints.ipynb). This is done using a
+collection of [Keras layers](tutorials/keras_layers.ipynb) that can satisfy
+constraints such as monotonicity, convexity and pairwise trust. The library also
+provides easy to setup [canned estimators](tutorials/canned_estimators.ipynb).
+
+## Concepts
+
+This section is a simplified version of the description in
+[Monotonic Calibrated Interpolated Look-Up Tables](http://jmlr.org/papers/v17/15-243.html)
+, JMLR 2016.
+
+### Lattices
+
+A *lattice* is an interpolated look-up table that can approximate arbitrary
+input-output relationships in your data. It overlaps a regular grid onto your
+input space and learns values for the output in the vertices of the grid. For a
+test point $x$, $f(x)$ is linearly interpolated from the lattice values
+surrounding $x$.
+
+
+
+The simple example above is a function with 2 input features and 4 parameters:
+$\theta=[0, 0.2, 0.4, 1]$, which are the function's values at the corners of the
+input space; the rest of the function is interpolated from these parameters.
+
+The function $f(x)$ can capture non-linear interactions between features. You
+can think of the lattice parameters as the height of poles set in the ground on
+a regular grid, and the resulting function is like cloth pulled tight against
+the four poles.
+
+With $D$ features and 2 vertices along each dimension, a regular lattice will
+have $2^D$ parameters. To fit a more flexible function, you can specify a
+finer-grained lattice over the feature space with more vertices along each
+dimension. Lattice regression functions are continuous and piecewise infinitely
+differentiable.
+
+### Calibration
+
+Let's say the preceding sample lattice represents a learned *user happiness*
+with a suggested local coffee shop calculated using features:
+
+* coffee price, in range 0 to 20 dollars
+* distance to the user, in range 0 to 30 kilometers
+
+We want our model to learn user happiness with a local coffee shop suggestion.
+TensorFlow Lattice models can use *piecewise linear functions* (with
+`tfl.layers.PWLCalibration`) to calibrate and normalize the input features to
+the range accepted by the lattice: 0.0 to 1.0 in the example lattice above. The
+following show examples such calibrations functions with 10 keypoints:
+
+![distance calibration](images/pwl_calibration_distance.png)
+![price calibration](images/pwl_calibration_price.png)
+
+It is often a good idea to use the quantiles of the features as input keypoints.
+TensorFlow Lattice [canned estimators](tutorials/canned_estimators.ipynb) can
+automatically set the input keypoints to the feature quantiles.
+
+For categorical features, TensorFlow Lattice provides categorical calibration
+(with `tfl.layers.CategoricalCalibration`) with similar output bounding to feed
+into a lattice.
+
+### Ensembles
+
+The number of parameters of a lattice layer increases exponentially with the
+number of input features, hence not scaling well to very high dimensions. To
+overcome this limitation, TensorFlow Lattice offers ensembles of lattices that
+combine (average) several *tiny* lattices, which enables the model to grow
+linearly in the number of features.
+
+The library provides two variations of these ensembles:
+
+* **Random Tiny Lattices** (RTL): Each submodel uses a random subset of
+ features (with replacement).
+
+* **Crystals** : The Crystals algorithm first trains a *prefitting* model that
+ estimates pairwise feature interactions. It then arranges the final ensemble
+ such that features with more non-linear interactions are in the same
+ lattices.
+
+## Why TensorFlow Lattice ?
+
+You can find a brief introduction to TensorFlow Lattice in
+[Google AI's Blog post](https://research.googleblog.com/).
+
+### Interpretability
+
+Since the parameters of each layer are the output of that layer, it is easy to
+analyze, understand and debug each part of the model.
+
+### Accurate and Flexible Models
+
+Using fine-grained lattices, you can get *arbitrarily complex* functions with a
+single lattice layer. Using multiple layers of calibrators and lattices often
+work nicely in practice and can match or outperform DNN models of similar sizes.
+
+### Common-Sense Shape Constraints
+
+Real world training data is often a somewhat biased representation of where the
+model will be applied.
+
+
+
+Unconstrained and flexible ML solutions such as DNNs or decision trees often act
+unexpectedly in parts of the input space not covered by the training data. Even
+though common forms of regularization can reduce nonsensical extrapolation, it
+is hardly enough to guarantee reasonable model behaviour across the entire input
+space.
+
+TensorFlow Lattice provides several types of *semantic regularization* through
+[shape constraints](tutorials/shape_constraints.ipynb):
+
+* **Monotonicity**: You can specify that the output should only
+ increase/decrease with respect to an input. In our example, you may want to
+ specify that increased distance to a coffee shop should only decrease the
+ predicted user preference.
+
+![linear fit](images/linear_fit.png) ![flexible fit](images/flexible_fit.png)
+![regularized fit](images/regularized_fit.png)
+![monotonic fit](images/monotonic_fit.png)
+
+* **Convexity/Concavity**: You can specify that the function shape can be
+ convex or concave. Mixed with monotonicity, this can force the function to
+ represent diminishing returns with respect to a given feature.
+
+* **Unimodality**: You can specify that the function should have a unique peak
+ or unique valley. This let you represent functions that have a *sweet spot*
+ with respect to a feature.
+
+* **Pairwise trust**: This constraint works on a pair of features and suggests
+ that one input feature semantically reflects trust in another feature. For
+ example, higher number of reviews makes you more confident in the average
+ star rating of a restaurant. The model will be more sensitive with respect
+ to the star rating (i.e. will have a larger slope with respect to the
+ rating) when the number of reviews is higher.
+
+### Controlled Flexibility with Regularizers
+
+In addition to shape constraints, TensorFlow lattice provides a number of
+regularizers to control the flexibility and smoothness of the function for each
+layer.
+
+* **Laplacian Regularizer**: Outputs of the lattice/calibration
+ vertices/keypoints are regularized towards the values of their respective
+ neighbors. This results in a *flatter* function.
+
+* **Hessian Regularizer**: This penalizes the first derivative of the PWL
+ calibration layer to make the function *more linear*.
+
+* **Wrinkle Regularizer**: This penalizes the second derivative of the PWL
+ calibration layer to avoid sudden changes in the curvature. It makes the
+ function smoother.
+
+* **Torsion**: Outputs of the lattice will be regularized towards preventing
+ torsion among the features. In other words, the model will be regularized
+ towards independence between the contributions of the features.
+
+### Mix and match with other Keras layers
+
+You can use TF Lattice layers in combination with other Keras layers to
+construct partially constrained or regularized models. For example, lattice or
+PWL calibration layers can be used at the last layer of deeper networks that
+include embeddings or other Keras layers.
+
+## Papers
+
+* [Shape Constraints for Set Functions](http://proceedings.mlr.press/v97/cotter19a.html),
+ Andrew Cotter, Maya Gupta, H. Jiang, Erez Louidor, Jim Muller, Taman
+ Narayan, Serena Wang, Tao Zhu. International Conference on Machine Learning
+ (ICML), 2019
+* [Diminishing Returns Shape Constraints for Interpretability and
+ Regularization](https://papers.nips.cc/paper/7916-diminishing-returns-shape-constraints-for-interpretability-and-regularization),
+ Maya Gupta, Dara Bahri, Andrew Cotter, Kevin Canini, Advances in Neural
+ Information Processing Systems (NeurIPS), 2018
+* [Deep Lattice Networks and Partial Monotonic Functions](https://research.google.com/pubs/pub46327.html),
+ Seungil You, Kevin Canini, David Ding, Jan Pfeifer, Maya R. Gupta, Advances
+ in Neural Information Processing Systems (NeurIPS), 2017
+* [Fast and Flexible Monotonic Functions with Ensembles of Lattices](https://papers.nips.cc/paper/6377-fast-and-flexible-monotonic-functions-with-ensembles-of-lattices),
+ Mahdi Milani Fard, Kevin Canini, Andrew Cotter, Jan Pfeifer, Maya Gupta,
+ Advances in Neural Information Processing Systems (NeurIPS), 2016
+* [Monotonic Calibrated Interpolated Look-Up Tables](http://jmlr.org/papers/v17/15-243.html),
+ Maya Gupta, Andrew Cotter, Jan Pfeifer, Konstantin Voevodski, Kevin Canini,
+ Alexander Mangylov, Wojciech Moczydlowski, Alexander van Esbroeck, Journal
+ of Machine Learning Research (JMLR), 2016
+* [Optimized Regression for Efficient Function Evaluation](http://ieeexplore.ieee.org/document/6203580/),
+ Eric Garcia, Raman Arora, Maya R. Gupta, IEEE Transactions on Image
+ Processing, 2012
+* [Lattice Regression](https://papers.nips.cc/paper/3694-lattice-regression),
+ Eric Garcia, Maya Gupta, Advances in Neural Information Processing Systems
+ (NeurIPS), 2009
+
+## Tutorials and API docs
+
+You can use [Canned Estimators](tutorials/canned_estimators.ipynb) or
+[Keras Layers](tutorials/keras_layers.ipynb). Check out
+[full API docs](api_docs/python/tfl.ipynb) for details.
diff --git a/docs/tutorials/canned_estimators.ipynb b/docs/tutorials/canned_estimators.ipynb
new file mode 100644
index 0000000..afd4bcf
--- /dev/null
+++ b/docs/tutorials/canned_estimators.ipynb
@@ -0,0 +1,722 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "7765UFHoyGx6"
+ },
+ "source": [
+ "##### Copyright 2020 The TensorFlow Authors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "form",
+ "colab": {},
+ "colab_type": "code",
+ "id": "KsOkK8O69PyT"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# https://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "ZS8z-_KeywY9"
+ },
+ "source": [
+ "# TF Lattice Canned Estimators"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "r61fkA2i9Y3_"
+ },
+ "source": [
+ "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/lattice/tutorials/canned_estimators\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/lattice/blob/master/docs/tutorials/canned_estimators.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/lattice/blob/master/docs/tutorials/canned_estimators.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/lattice/tutorials/canned_estimators.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ "\u003c/table\u003e"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "WCpl-9WDVq9d"
+ },
+ "source": [
+ "## Overview\n",
+ "\n",
+ "Canned estimators are quick and easy ways to train TFL models for typical use cases. This guide outlines the steps needed to create a TFL canned estimator."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "x769lI12IZXB"
+ },
+ "source": [
+ "## Setup"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "fbBVAR6UeRN5"
+ },
+ "source": [
+ "Installing TF Lattice package:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "bpXjJKpSd3j4"
+ },
+ "outputs": [],
+ "source": [
+ "#@test {\"skip\": true}\n",
+ "!pip install tensorflow-lattice"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "jSVl9SHTeSGX"
+ },
+ "source": [
+ "Importing required packages:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "both",
+ "colab": {},
+ "colab_type": "code",
+ "id": "FbZDk8bIx8ig"
+ },
+ "outputs": [],
+ "source": [
+ "from __future__ import absolute_import, division, print_function, unicode_literals\n",
+ "\n",
+ "try:\n",
+ " # %tensorflow_version only exists in Colab.\n",
+ " %tensorflow_version 2.x\n",
+ "except Exception:\n",
+ " pass\n",
+ "import tensorflow as tf\n",
+ "\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import tensorflow_lattice as tfl\n",
+ "from tensorflow import feature_column as fc"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "svPuM6QNxlrH"
+ },
+ "source": [
+ "Downloading the UCI Statlog (Heart) dataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "both",
+ "colab": {},
+ "colab_type": "code",
+ "id": "j-k1qTR_yvBl"
+ },
+ "outputs": [],
+ "source": [
+ "csv_file = tf.keras.utils.get_file(\n",
+ " 'heart.csv', 'http://storage.googleapis.com/applied-dl/heart.csv')\n",
+ "df = pd.read_csv(csv_file)\n",
+ "target = df.pop('target')\n",
+ "train_size = int(len(df) * 0.8)\n",
+ "train_x = df[:train_size]\n",
+ "train_y = target[:train_size]\n",
+ "test_x = df[train_size:]\n",
+ "test_y = target[train_size:]\n",
+ "df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "nKkAw12SxvGG"
+ },
+ "source": [
+ "Setting the default values used for training in this guide:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "both",
+ "colab": {},
+ "colab_type": "code",
+ "id": "1T6GFI9F6mcG"
+ },
+ "outputs": [],
+ "source": [
+ "LEARNING_RATE = 0.01\n",
+ "BATCH_SIZE = 128\n",
+ "NUM_EPOCHS = 500\n",
+ "PREFITTING_NUM_EPOCHS = 10"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "0TGfzhPHzpix"
+ },
+ "source": [
+ "## Feature Columns\n",
+ "\n",
+ "As for any other TF estimator, data needs to be passed to the estimator, which is typically via an input_fn and parsed using [FeatureColumns](https://www.tensorflow.org/guide/feature_columns)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "DCIUz8apzs0l"
+ },
+ "outputs": [],
+ "source": [
+ "# Feature columns.\n",
+ "# - age\n",
+ "# - sex\n",
+ "# - cp chest pain type (4 values)\n",
+ "# - trestbps resting blood pressure\n",
+ "# - chol serum cholestoral in mg/dl\n",
+ "# - fbs fasting blood sugar \u003e 120 mg/dl\n",
+ "# - restecg resting electrocardiographic results (values 0,1,2)\n",
+ "# - thalach maximum heart rate achieved\n",
+ "# - exang exercise induced angina\n",
+ "# - oldpeak ST depression induced by exercise relative to rest\n",
+ "# - slope the slope of the peak exercise ST segment\n",
+ "# - ca number of major vessels (0-3) colored by flourosopy\n",
+ "# - thal 3 = normal; 6 = fixed defect; 7 = reversable defect\n",
+ "feature_columns = [\n",
+ " fc.numeric_column('age', default_value=-1),\n",
+ " fc.categorical_column_with_vocabulary_list('sex', [0, 1]),\n",
+ " fc.numeric_column('cp'),\n",
+ " fc.numeric_column('trestbps', default_value=-1),\n",
+ " fc.numeric_column('chol'),\n",
+ " fc.categorical_column_with_vocabulary_list('fbs', [0, 1]),\n",
+ " fc.categorical_column_with_vocabulary_list('restecg', [0, 1, 2]),\n",
+ " fc.numeric_column('thalach'),\n",
+ " fc.categorical_column_with_vocabulary_list('exang', [0, 1]),\n",
+ " fc.numeric_column('oldpeak'),\n",
+ " fc.categorical_column_with_vocabulary_list('slope', [0, 1, 2]),\n",
+ " fc.numeric_column('ca'),\n",
+ " fc.categorical_column_with_vocabulary_list(\n",
+ " 'thal', ['normal', 'fixed', 'reversible']),\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "hEZstmtT2CA3"
+ },
+ "source": [
+ "TFL canned estimators use the type of the feature column to decide what type of calibration layer to use. We use a `tfl.layers.PWLCalibration` layer for numeric feature columns and a `tfl.layers.CategoricalCalibration` layer for categorical feature columns.\n",
+ "\n",
+ "Note that categorical feature columns are not wrapped by an embedding feature column. They are directly fed into the estimator."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "H_LoW_9m5OFL"
+ },
+ "source": [
+ "## Creating input_fn\n",
+ "\n",
+ "As for any other estimator, you can use an input_fn to feed data to the model for training and evaluation. TFL estimators can automatically calculate quantiles of the features and use them as input keypoints for the PWL calibration layer. To do so, they require passing a `feature_analysis_input_fn`, which is similar to the training input_fn but with a single epoch or a subsample of the data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "lFVy1Efy5NKD"
+ },
+ "outputs": [],
+ "source": [
+ "train_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(\n",
+ " x=train_x,\n",
+ " y=train_y,\n",
+ " shuffle=False,\n",
+ " batch_size=BATCH_SIZE,\n",
+ " num_epochs=NUM_EPOCHS,\n",
+ " num_threads=1)\n",
+ "\n",
+ "# feature_analysis_input_fn is used to collect statistics about the input.\n",
+ "feature_analysis_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(\n",
+ " x=train_x,\n",
+ " y=train_y,\n",
+ " shuffle=False,\n",
+ " batch_size=BATCH_SIZE,\n",
+ " # Note that we only need one pass over the data.\n",
+ " num_epochs=1,\n",
+ " num_threads=1)\n",
+ "\n",
+ "test_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(\n",
+ " x=test_x,\n",
+ " y=test_y,\n",
+ " shuffle=False,\n",
+ " batch_size=BATCH_SIZE,\n",
+ " num_epochs=1,\n",
+ " num_threads=1)\n",
+ "\n",
+ "# Serving input fn is used to create saved models.\n",
+ "serving_input_fn = (\n",
+ " tf.estimator.export.build_parsing_serving_input_receiver_fn(\n",
+ " feature_spec=fc.make_parse_example_spec(feature_columns)))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "uQlzREcm2Wbj"
+ },
+ "source": [
+ "## Feature Configs\n",
+ "\n",
+ "Feature calibration and per-feature configurations are set using `tfl.configs.FeatureConfig`. Feature configurations include monotonicity constraints, per-feature regularization (see `tfl.configs.RegularizerConfig`), and lattice sizes for lattice models.\n",
+ "\n",
+ "If no configuration is defined for an input feature, the default configuration in `tfl.config.FeatureConfig` is used."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "vD0tNpiO3p9c"
+ },
+ "outputs": [],
+ "source": [
+ "# Feature configs are used to specify how each feature is calibrated and used.\n",
+ "feature_configs = [\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name='age',\n",
+ " lattice_size=3,\n",
+ " # By default, input keypoints of pwl are quantiles of the feature.\n",
+ " pwl_calibration_num_keypoints=5,\n",
+ " monotonicity='increasing',\n",
+ " pwl_calibration_clip_max=100,\n",
+ " # Per feature regularization.\n",
+ " regularizer_configs=[\n",
+ " tfl.configs.RegularizerConfig(name='calib_wrinkle', l2=0.1),\n",
+ " ],\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name='cp',\n",
+ " pwl_calibration_num_keypoints=4,\n",
+ " # Keypoints can be uniformly spaced.\n",
+ " pwl_calibration_input_keypoints='uniform',\n",
+ " monotonicity='increasing',\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name='chol',\n",
+ " # Explicit input keypoint initialization.\n",
+ " pwl_calibration_input_keypoints=[126.0, 210.0, 247.0, 286.0, 564.0],\n",
+ " monotonicity='increasing',\n",
+ " pwl_calibration_clip_min=130,\n",
+ " # Calibration can be forced to span the full output range by clamping.\n",
+ " pwl_calibration_clamp_min=True,\n",
+ " pwl_calibration_clamp_max=True,\n",
+ " # Per feature regularization.\n",
+ " regularizer_configs=[\n",
+ " tfl.configs.RegularizerConfig(name='calib_hessian', l2=1e-4),\n",
+ " ],\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name='fbs',\n",
+ " # Partial monotonicity: output(0) \u003c= output(1)\n",
+ " monotonicity=[(0, 1)],\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name='trestbps',\n",
+ " pwl_calibration_num_keypoints=5,\n",
+ " monotonicity='decreasing',\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name='thalach',\n",
+ " pwl_calibration_num_keypoints=5,\n",
+ " monotonicity='decreasing',\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name='restecg',\n",
+ " # Partial monotonicity: output(0) \u003c= output(1), output(0) \u003c= output(2)\n",
+ " monotonicity=[(0, 1), (0, 2)],\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name='exang',\n",
+ " # Partial monotonicity: output(0) \u003c= output(1)\n",
+ " monotonicity=[(0, 1)],\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name='oldpeak',\n",
+ " pwl_calibration_num_keypoints=5,\n",
+ " monotonicity='increasing',\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name='slope',\n",
+ " # Partial monotonicity: output(0) \u003c= output(1), output(0) \u003c= output(2)\n",
+ " monotonicity=[(0, 1), (1, 2)],\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name='ca',\n",
+ " pwl_calibration_num_keypoints=4,\n",
+ " monotonicity='increasing',\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name='thal',\n",
+ " # Partial monotonicity:\n",
+ " # output(normal) \u003c= output(fixed)\n",
+ " # output(normal) \u003c= output(reversible) \n",
+ " monotonicity=[('normal', 'fixed'), ('normal', 'reversible')],\n",
+ " ),\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "LKBULveZ4mr3"
+ },
+ "source": [
+ "## Calibrated Linear Model\n",
+ "\n",
+ "To construct a TFL canned estimator, construct a model configuration from `tfl.configs`. A calibrated linear model is constructed using `tfl.configs.CalibratedLinearConfig`. It applies piecewise-linear and categorical calibration on the input features, followed by a linear combination and an optional output piecewise-linear calibration. When using output calibration or when output bounds are specified, the linear layer will apply weighted averaging on calibrated inputs.\n",
+ "\n",
+ "This example creates a calibrated linear model on the first 5 features. We use\n",
+ "`tfl.visualization` to plot the model graph with the calibrator plots."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "diRRozio4sAL"
+ },
+ "outputs": [],
+ "source": [
+ "# Model config defines the model strcutre for the estimator.\n",
+ "model_config = tfl.configs.CalibratedLinearConfig(\n",
+ " feature_configs=feature_configs,\n",
+ " use_bias=True,\n",
+ " output_calibration=True,\n",
+ " regularizer_configs=[\n",
+ " # Regularizer for the output calibrator.\n",
+ " tfl.configs.RegularizerConfig(name='output_calib_hessian', l2=1e-4),\n",
+ " ])\n",
+ "# A CannedClassifier is constructed from the given model config.\n",
+ "estimator = tfl.estimators.CannedClassifier(\n",
+ " feature_columns=feature_columns[:5],\n",
+ " model_config=model_config,\n",
+ " feature_analysis_input_fn=feature_analysis_input_fn,\n",
+ " optimizer=tf.keras.optimizers.Adam(LEARNING_RATE),\n",
+ " config=tf.estimator.RunConfig(tf_random_seed=42))\n",
+ "estimator.train(input_fn=train_input_fn)\n",
+ "results = estimator.evaluate(input_fn=test_input_fn)\n",
+ "print('Calibrated linear test AUC: {}'.format(results['auc']))\n",
+ "saved_model_path = estimator.export_saved_model(estimator.model_dir,\n",
+ " serving_input_fn)\n",
+ "model_graph = tfl.estimators.get_model_graph(saved_model_path)\n",
+ "tfl.visualization.draw_model_graph(model_graph)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "zWzPM2_p977t"
+ },
+ "source": [
+ "## Calibrated Lattice Model\n",
+ "\n",
+ "A calibrated lattice model is constructed using `tfl.configs.CalibratedLatticeConfig`. A calibrated lattice model applies piecewise-linear and categorical calibration on the input features, followed by a lattice model and an optional output piecewise-linear calibration.\n",
+ "\n",
+ "This example creates a calibrated lattice model on the first 5 features.\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "C6EvVpKW4BbC"
+ },
+ "outputs": [],
+ "source": [
+ "# This is calibrated lattice model: Inputs are calibrated, then combined\n",
+ "# non-linearly using a lattice layer.\n",
+ "model_config = tfl.configs.CalibratedLatticeConfig(\n",
+ " feature_configs=feature_configs,\n",
+ " regularizer_configs=[\n",
+ " # Torsion regularizer applied to the lattice to make it more linear.\n",
+ " tfl.configs.RegularizerConfig(name='torsion', l2=1e-4),\n",
+ " # Globally defined calibration regularizer is applied to all features.\n",
+ " tfl.configs.RegularizerConfig(name='calib_hessian', l2=1e-4),\n",
+ " ])\n",
+ "# A CannedClassifier is constructed from the given model config.\n",
+ "estimator = tfl.estimators.CannedClassifier(\n",
+ " feature_columns=feature_columns[:5],\n",
+ " model_config=model_config,\n",
+ " feature_analysis_input_fn=feature_analysis_input_fn,\n",
+ " optimizer=tf.keras.optimizers.Adam(LEARNING_RATE),\n",
+ " config=tf.estimator.RunConfig(tf_random_seed=42))\n",
+ "estimator.train(input_fn=train_input_fn)\n",
+ "results = estimator.evaluate(input_fn=test_input_fn)\n",
+ "print('Calibrated lattice test AUC: {}'.format(results['auc']))\n",
+ "saved_model_path = estimator.export_saved_model(estimator.model_dir,\n",
+ " serving_input_fn)\n",
+ "model_graph = tfl.estimators.get_model_graph(saved_model_path)\n",
+ "tfl.visualization.draw_model_graph(model_graph)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "9494K_ZBKFcm"
+ },
+ "source": [
+ "## Calibrated Lattice Ensemble\n",
+ "\n",
+ "When the number of features is large, you can use an ensemble model, which creates multiple smaller lattices for subsets of the features and averages their output instead of creating just a single huge lattice. Ensemble lattice models are constructed using `tfl.configs.CalibratedLatticeEnsembleConfig`. A calibrated lattice ensemble model applies piecewise-linear and categorical calibration on the input feature, followed by an ensemble of lattice models and an optional output piecewise-linear calibration.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "KjrzziMFKuCB"
+ },
+ "source": [
+ "### Random Lattice Ensemble\n",
+ "\n",
+ "The following model config uses a random subset of features for each lattice."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "YBSS7dLjKExq"
+ },
+ "outputs": [],
+ "source": [
+ "# This is random lattice ensemble model with separate calibration:\n",
+ "# model output is the average output of separatly calibrated lattices.\n",
+ "model_config = tfl.configs.CalibratedLatticeEnsembleConfig(\n",
+ " feature_configs=feature_configs,\n",
+ " num_lattices=5,\n",
+ " lattice_rank=3)\n",
+ "# A CannedClassifier is constructed from the given model config.\n",
+ "estimator = tfl.estimators.CannedClassifier(\n",
+ " feature_columns=feature_columns,\n",
+ " model_config=model_config,\n",
+ " feature_analysis_input_fn=feature_analysis_input_fn,\n",
+ " optimizer=tf.keras.optimizers.Adam(LEARNING_RATE),\n",
+ " config=tf.estimator.RunConfig(tf_random_seed=42))\n",
+ "estimator.train(input_fn=train_input_fn)\n",
+ "results = estimator.evaluate(input_fn=test_input_fn)\n",
+ "print('Random ensemble test AUC: {}'.format(results['auc']))\n",
+ "saved_model_path = estimator.export_saved_model(estimator.model_dir,\n",
+ " serving_input_fn)\n",
+ "model_graph = tfl.estimators.get_model_graph(saved_model_path)\n",
+ "tfl.visualization.draw_model_graph(model_graph, calibrator_dpi=15)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "LSXEaYAULRvf"
+ },
+ "source": [
+ "### Crystals Lattice Ensemble\n",
+ "\n",
+ "TFL also provides a heuristic feature arrangement algorithm, called *Crystals*. The Crystals algorithm first trains a *prefitting model* that estimates pairwise feature interactions. It then arranges the final ensemble such that features with more non-linear interactions are in the same lattices.\n",
+ "\n",
+ "For Crystals models, you will also need to provide a `prefitting_input_fn` that is used to train the prefitting model, as described above. The prefitting model does not need to be fully trained, so a few epochs should be enough.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "FjQKh9saMaFu"
+ },
+ "outputs": [],
+ "source": [
+ "prefitting_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(\n",
+ " x=train_x,\n",
+ " y=train_y,\n",
+ " shuffle=False,\n",
+ " batch_size=BATCH_SIZE,\n",
+ " num_epochs=PREFITTING_NUM_EPOCHS,\n",
+ " num_threads=1)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "fVnZpwX8MtPi"
+ },
+ "source": [
+ "You can then create a Crystal model by setting `lattice='crystals'` in the model config."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "f4awRMDe-eMv"
+ },
+ "outputs": [],
+ "source": [
+ "# This is Crystals ensemble model with separate calibration: model output is\n",
+ "# the average output of separatly calibrated lattices.\n",
+ "model_config = tfl.configs.CalibratedLatticeEnsembleConfig(\n",
+ " feature_configs=feature_configs,\n",
+ " lattices='crystals',\n",
+ " num_lattices=5,\n",
+ " lattice_rank=3)\n",
+ "# A CannedClassifier is constructed from the given model config.\n",
+ "estimator = tfl.estimators.CannedClassifier(\n",
+ " feature_columns=feature_columns,\n",
+ " model_config=model_config,\n",
+ " feature_analysis_input_fn=feature_analysis_input_fn,\n",
+ " # prefitting_input_fn is required to train the prefitting model.\n",
+ " prefitting_input_fn=prefitting_input_fn,\n",
+ " optimizer=tf.keras.optimizers.Adam(LEARNING_RATE),\n",
+ " prefitting_optimizer=tf.keras.optimizers.Adam(LEARNING_RATE),\n",
+ " config=tf.estimator.RunConfig(tf_random_seed=42))\n",
+ "estimator.train(input_fn=train_input_fn)\n",
+ "results = estimator.evaluate(input_fn=test_input_fn)\n",
+ "print('Crystals ensemble test AUC: {}'.format(results['auc']))\n",
+ "saved_model_path = estimator.export_saved_model(estimator.model_dir,\n",
+ " serving_input_fn)\n",
+ "model_graph = tfl.estimators.get_model_graph(saved_model_path)\n",
+ "tfl.visualization.draw_model_graph(model_graph, calibrator_dpi=15)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "Isb2vyLAVBM1"
+ },
+ "source": [
+ "You can plot feature calibrators with more details using the `tfl.visualization` module."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "DJPaREuWS2sg"
+ },
+ "outputs": [],
+ "source": [
+ "_ = tfl.visualization.plot_feature_calibrator(model_graph, \"age\")\n",
+ "_ = tfl.visualization.plot_feature_calibrator(model_graph, \"restecg\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [],
+ "name": "tfl_canned_estimators.ipynb",
+ "private_outputs": true,
+ "provenance": [
+ {
+ "file_id": "1gw3igUWesgUCASoPM-xRZk6bGg3E1qOX",
+ "timestamp": 1579554854035
+ }
+ ],
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/docs/tutorials/custom_estimators.ipynb b/docs/tutorials/custom_estimators.ipynb
new file mode 100644
index 0000000..3850d5a
--- /dev/null
+++ b/docs/tutorials/custom_estimators.ipynb
@@ -0,0 +1,443 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "7765UFHoyGx6"
+ },
+ "source": [
+ "##### Copyright 2020 The TensorFlow Authors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "form",
+ "colab": {},
+ "colab_type": "code",
+ "id": "KsOkK8O69PyT"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# https://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "ZS8z-_KeywY9"
+ },
+ "source": [
+ "# TF Lattice Custom Estimators"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "r61fkA2i9Y3_"
+ },
+ "source": [
+ "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/lattice/tutorials/custom_estimators\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/lattice/blob/master/docs/tutorials/custom_estimators.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/lattice/blob/master/docs/tutorials/custom_estimators.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/lattice/tutorials/custom_estimators.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ "\u003c/table\u003e"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "Ur6yCw7YVvr8"
+ },
+ "source": [
+ "## Overview\n",
+ "\n",
+ "You can use custom estimators to create arbitrarily monotonic models using TFL layers. This guide outlines the steps needed to create such estimators."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "x769lI12IZXB"
+ },
+ "source": [
+ "## Setup"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "fbBVAR6UeRN5"
+ },
+ "source": [
+ "Installing TF Lattice package:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "bpXjJKpSd3j4"
+ },
+ "outputs": [],
+ "source": [
+ "#@test {\"skip\": true}\n",
+ "!pip install tensorflow-lattice"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "jSVl9SHTeSGX"
+ },
+ "source": [
+ "Importing required packages:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "both",
+ "colab": {},
+ "colab_type": "code",
+ "id": "P9rMpg1-ASY3"
+ },
+ "outputs": [],
+ "source": [
+ "from __future__ import absolute_import, division, print_function, unicode_literals\n",
+ "!pip install tensorflow-lattice\n",
+ "\n",
+ "try:\n",
+ " # %tensorflow_version only exists in Colab.\n",
+ " %tensorflow_version 2.x\n",
+ "except Exception:\n",
+ " pass\n",
+ "import tensorflow as tf\n",
+ "\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import tensorflow_lattice as tfl\n",
+ "from tensorflow import feature_column as fc\n",
+ "\n",
+ "from tensorflow_estimator.python.estimator.canned import optimizers\n",
+ "from tensorflow_estimator.python.estimator.head import binary_class_head"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "svPuM6QNxlrH"
+ },
+ "source": [
+ "Downloading the UCI Statlog (Heart) dataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "both",
+ "colab": {},
+ "colab_type": "code",
+ "id": "M0CmH1gPASZF"
+ },
+ "outputs": [],
+ "source": [
+ "csv_file = tf.keras.utils.get_file(\n",
+ " 'heart.csv', 'http://storage.googleapis.com/applied-dl/heart.csv')\n",
+ "df = pd.read_csv(csv_file)\n",
+ "target = df.pop('target')\n",
+ "train_size = int(len(df) * 0.8)\n",
+ "train_x = df[:train_size]\n",
+ "train_y = target[:train_size]\n",
+ "test_x = df[train_size:]\n",
+ "test_y = target[train_size:]\n",
+ "df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "nKkAw12SxvGG"
+ },
+ "source": [
+ "Setting the default values used for training in this guide:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "both",
+ "colab": {},
+ "colab_type": "code",
+ "id": "1T6GFI9F6mcG"
+ },
+ "outputs": [],
+ "source": [
+ "LEARNING_RATE = 0.1\n",
+ "BATCH_SIZE = 128\n",
+ "NUM_EPOCHS = 1000"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "0TGfzhPHzpix"
+ },
+ "source": [
+ "## Feature Columns\n",
+ "\n",
+ "As for any other TF estimator, data needs to be passed to the estimator, which is typically via an input_fn and parsed using [FeatureColumns](https://www.tensorflow.org/guide/feature_columns)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "DCIUz8apzs0l"
+ },
+ "outputs": [],
+ "source": [
+ "# Feature columns.\n",
+ "# - age\n",
+ "# - sex\n",
+ "# - ca number of major vessels (0-3) colored by flourosopy\n",
+ "# - thal 3 = normal; 6 = fixed defect; 7 = reversable defect\n",
+ "feature_columns = [\n",
+ " fc.numeric_column('age', default_value=-1),\n",
+ " fc.categorical_column_with_vocabulary_list('sex', [0, 1]),\n",
+ " fc.numeric_column('ca'),\n",
+ " fc.categorical_column_with_vocabulary_list(\n",
+ " 'thal', ['normal', 'fixed', 'reversible']),\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "hEZstmtT2CA3"
+ },
+ "source": [
+ "Note that categorical features do not need to be wrapped by a dense feature column, since `tfl.laysers.CategoricalCalibration` layer can directly consume category indices."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "H_LoW_9m5OFL"
+ },
+ "source": [
+ "## Creating input_fn\n",
+ "\n",
+ "As for any other estimator, you can use input_fn to feed data to the model for training and evaluation."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "lFVy1Efy5NKD"
+ },
+ "outputs": [],
+ "source": [
+ "train_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(\n",
+ " x=train_x,\n",
+ " y=train_y,\n",
+ " shuffle=True,\n",
+ " batch_size=BATCH_SIZE,\n",
+ " num_epochs=NUM_EPOCHS,\n",
+ " num_threads=1)\n",
+ "\n",
+ "test_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(\n",
+ " x=test_x,\n",
+ " y=test_y,\n",
+ " shuffle=False,\n",
+ " batch_size=BATCH_SIZE,\n",
+ " num_epochs=1,\n",
+ " num_threads=1)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "kbrgSr9KaRg0"
+ },
+ "source": [
+ "## Creating model_fn\n",
+ "\n",
+ "There are several ways to create a custom estimator. Here we will construct a `model_fn` that calls a Keras model on the parsed input tensors. To parse the input features, you can use `tf.feature_column.input_layer`, `tf.keras.layers.DenseFeatures`, or `tfl.estimators.transform_features`. If you use the latter, you will not need to wrap categorical features with dense feature columns, and the resulting tensors will not be concatenated, which makes it easier to use the features in the calibration layers.\n",
+ "\n",
+ "To construct a model, you can mix and match TFL layers or any other Keras layers. Here we create a calibrated lattice Keras model out of TFL layers and impose several monotonicity constraints. When then use the Keras model to create the custom estimator.\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "n2Zrv6OPaQO2"
+ },
+ "outputs": [],
+ "source": [
+ "def model_fn(features, labels, mode, config):\n",
+ " \"\"\"model_fn for the custom estimator.\"\"\"\n",
+ " del config\n",
+ " input_tensors = tfl.estimators.transform_features(features, feature_columns)\n",
+ " inputs = {\n",
+ " key: tf.keras.layers.Input(shape=(1,), name=key) for key in input_tensors\n",
+ " }\n",
+ "\n",
+ " lattice_sizes = [3, 2, 2, 2]\n",
+ " lattice_monotonicities = ['increasing', 'none', 'increasing', 'increasing']\n",
+ " lattice_input = tf.keras.layers.Concatenate(axis=1)([\n",
+ " tfl.layers.PWLCalibration(\n",
+ " input_keypoints=np.linspace(10, 100, num=8, dtype=np.float32),\n",
+ " # The output range of the calibrator should be the input range of\n",
+ " # the following lattice dimension.\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[0] - 1.0,\n",
+ " monotonicity='increasing',\n",
+ " )(inputs['age']),\n",
+ " tfl.layers.CategoricalCalibration(\n",
+ " # Number of categories including any missing/default category.\n",
+ " num_buckets=2,\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[1] - 1.0,\n",
+ " )(inputs['sex']),\n",
+ " tfl.layers.PWLCalibration(\n",
+ " input_keypoints=[0.0, 1.0, 2.0, 3.0],\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[0] - 1.0,\n",
+ " # You can specify TFL regularizers as tuple\n",
+ " # ('regularizer name', l1, l2).\n",
+ " kernel_regularizer=('hessian', 0.0, 1e-4),\n",
+ " monotonicity='increasing',\n",
+ " )(inputs['ca']),\n",
+ " tfl.layers.CategoricalCalibration(\n",
+ " num_buckets=3,\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[1] - 1.0,\n",
+ " # Categorical monotonicity can be partial order.\n",
+ " # (i, j) indicates that we must have output(i) \u003c= output(j).\n",
+ " # Make sure to set the lattice monotonicity to 'increasing' for this\n",
+ " # dimension.\n",
+ " monotonicities=[(0, 1), (0, 2)],\n",
+ " )(inputs['thal']),\n",
+ " ])\n",
+ " output = tfl.layers.Lattice(\n",
+ " lattice_sizes=lattice_sizes, monotonicities=lattice_monotonicities)(\n",
+ " lattice_input)\n",
+ "\n",
+ " training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
+ " model = tf.keras.Model(inputs=inputs, outputs=output)\n",
+ " logits = model(input_tensors, training=training)\n",
+ "\n",
+ " if training:\n",
+ " optimizer = optimizers.get_optimizer_instance_v2('Adagrad', LEARNING_RATE)\n",
+ " else:\n",
+ " optimizer = None\n",
+ "\n",
+ " head = binary_class_head.BinaryClassHead()\n",
+ " return head.create_estimator_spec(\n",
+ " features=features,\n",
+ " mode=mode,\n",
+ " labels=labels,\n",
+ " optimizer=optimizer,\n",
+ " logits=logits,\n",
+ " trainable_variables=model.trainable_variables,\n",
+ " update_ops=model.updates)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "mng-VtsSbVtQ"
+ },
+ "source": [
+ "## Training and Estimator\n",
+ "\n",
+ "Using the `model_fn` we can create and train the estimator."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "j38GaEbKbZju"
+ },
+ "outputs": [],
+ "source": [
+ "estimator = tf.estimator.Estimator(model_fn=model_fn)\n",
+ "estimator.train(input_fn=train_input_fn)\n",
+ "results = estimator.evaluate(input_fn=test_input_fn)\n",
+ "print('AUC: {}'.format(results['auc']))"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [],
+ "name": "tfl_custom_estimators.ipynb",
+ "private_outputs": true,
+ "provenance": [
+ {
+ "file_id": "1YQhpyfKAW4Gz49gDFMJtVSpAM-Zi12h9",
+ "timestamp": 1579559437099
+ }
+ ],
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/docs/tutorials/keras_layers.ipynb b/docs/tutorials/keras_layers.ipynb
new file mode 100644
index 0000000..6d96515
--- /dev/null
+++ b/docs/tutorials/keras_layers.ipynb
@@ -0,0 +1,838 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "7765UFHoyGx6"
+ },
+ "source": [
+ "##### Copyright 2020 The TensorFlow Authors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "form",
+ "colab": {},
+ "colab_type": "code",
+ "id": "KsOkK8O69PyT"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# https://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "ZS8z-_KeywY9"
+ },
+ "source": [
+ "# Creating Keras Models with TFL Layers"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "r61fkA2i9Y3_"
+ },
+ "source": [
+ "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/lattice/tutorials/keras_layers\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/lattice/blob/master/docs/tutorials/keras_layers.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/lattice/blob/master/docs/tutorials/keras_layers.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/lattice/tutorials/keras_layers.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ "\u003c/table\u003e"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "ecLbJCvJSSCd"
+ },
+ "source": [
+ "##Overview\n",
+ "\n",
+ "You can use TFL Keras layers to construct Keras models with monotonicity and other shape constraints. This example builds and trains a calibrated lattice model for the UCI heart dataset using TFL layers.\n",
+ "\n",
+ "In a calibrated lattice model, each feature is transformed by a `tfl.layers.PWLCalibration` or a `tfl.layers.CategoricalCalibration` layer and the results are nonlinearly fused using a `tfl.layers.Lattice`."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "x769lI12IZXB"
+ },
+ "source": [
+ "## Setup"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "fbBVAR6UeRN5"
+ },
+ "source": [
+ "Installing TF Lattice package:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "bpXjJKpSd3j4"
+ },
+ "outputs": [],
+ "source": [
+ "#@test {\"skip\": true}\n",
+ "!pip install tensorflow-lattice"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "jSVl9SHTeSGX"
+ },
+ "source": [
+ "Importing required packages:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "both",
+ "colab": {},
+ "colab_type": "code",
+ "id": "pm0LD8iyIZXF"
+ },
+ "outputs": [],
+ "source": [
+ "from __future__ import absolute_import, division, print_function, unicode_literals\n",
+ "\n",
+ "try:\n",
+ " # %tensorflow_version only exists in Colab.\n",
+ " %tensorflow_version 2.x\n",
+ "except Exception:\n",
+ " pass\n",
+ "import tensorflow as tf\n",
+ "\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import tensorflow_lattice as tfl\n",
+ "from tensorflow import feature_column as fc"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "svPuM6QNxlrH"
+ },
+ "source": [
+ "Downloading the UCI Statlog (Heart) dataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "both",
+ "colab": {},
+ "colab_type": "code",
+ "id": "PG3pFtK-IZXM"
+ },
+ "outputs": [],
+ "source": [
+ "# UCI Statlog (Heart) dataset.\n",
+ "csv_file = tf.keras.utils.get_file(\n",
+ " 'heart.csv', 'http://storage.googleapis.com/applied-dl/heart.csv')\n",
+ "training_data_df = pd.read_csv(csv_file).sample(\n",
+ " frac=1.0, random_state=41).reset_index(drop=True)\n",
+ "training_data_df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "nKkAw12SxvGG"
+ },
+ "source": [
+ "Setting the default values used for training in this guide:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "both",
+ "colab": {},
+ "colab_type": "code",
+ "id": "krAJBE-yIZXR"
+ },
+ "outputs": [],
+ "source": [
+ "LEARNING_RATE = 0.1\n",
+ "BATCH_SIZE = 128\n",
+ "NUM_EPOCHS = 100"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "0TGfzhPHzpix"
+ },
+ "source": [
+ "## Sequential Keras Model\n",
+ "\n",
+ "This example creates a Sequential Keras model and only uses TFL layers.\n",
+ "\n",
+ "Lattice layers expect `input[i]` to be within `[0, lattice_sizes[i] - 1.0]`, so we need to define the lattice sizes ahead of the calibration layers so we can properly specify output range of the calibration layers.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "nOQWqPAbQS3o"
+ },
+ "outputs": [],
+ "source": [
+ "# Lattice layer expects input[i] to be within [0, lattice_sizes[i] - 1.0], so\n",
+ "lattice_sizes = [3, 2, 2, 2, 2, 2, 2]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "W3DnEKWvQYXm"
+ },
+ "source": [
+ "We use a `tfl.layers.ParallelCombination` layer to group together calibration layers which have to be executed in paralel in order to be able to create a Sequential model.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "o_hyk5GkQfl8"
+ },
+ "outputs": [],
+ "source": [
+ "combined_calibrators = tfl.layers.ParallelCombination()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "BPZsSUZiQiwc"
+ },
+ "source": [
+ "We create a calibration layer for each feature and add it to the parallel combination layer. For numeric features we use `tfl.layers.PWLCalibration` and for categorical features we use `tfl.layers.CategoricalCalibration`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "DXPc6rSGxzFZ"
+ },
+ "outputs": [],
+ "source": [
+ "# ############### age ###############\n",
+ "calibrator = tfl.layers.PWLCalibration(\n",
+ " # Every PWLCalibration layer must have keypoints of piecewise linear\n",
+ " # function specified. Easiest way to specify them is to uniformly cover\n",
+ " # entire input range by using numpy.linspace().\n",
+ " input_keypoints=np.linspace(\n",
+ " training_data_df['age'].min(), training_data_df['age'].max(), num=5),\n",
+ " # You need to ensure that input keypoints have same dtype as layer input.\n",
+ " # You can do it by setting dtype here or by providing keypoints in such\n",
+ " # format which will be converted to deisred tf.dtype by default.\n",
+ " dtype=tf.float32,\n",
+ " # Output range must correspond to expected lattice input range.\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[0] - 1.0,\n",
+ ")\n",
+ "combined_calibrators.append(calibrator)\n",
+ "\n",
+ "# ############### sex ###############\n",
+ "# For boolean features simply specify CategoricalCalibration layer with 2\n",
+ "# buckets.\n",
+ "calibrator = tfl.layers.CategoricalCalibration(\n",
+ " num_buckets=2,\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[1] - 1.0,\n",
+ " # Initializes all outputs to (output_min + output_max) / 2.0.\n",
+ " kernel_initializer='constant')\n",
+ "combined_calibrators.append(calibrator)\n",
+ "\n",
+ "# ############### cp ###############\n",
+ "calibrator = tfl.layers.PWLCalibration(\n",
+ " # Here instead of specifying dtype of layer we convert keypoints into\n",
+ " # np.float32.\n",
+ " input_keypoints=np.linspace(1, 4, num=4, dtype=np.float32),\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[2] - 1.0,\n",
+ " monotonicity='increasing',\n",
+ " # You can specify TFL regularizers as a tuple ('regularizer name', l1, l2).\n",
+ " kernel_regularizer=('hessian', 0.0, 1e-4))\n",
+ "combined_calibrators.append(calibrator)\n",
+ "\n",
+ "# ############### trestbps ###############\n",
+ "calibrator = tfl.layers.PWLCalibration(\n",
+ " # Alternatively, you might want to use quantiles as keypoints instead of\n",
+ " # uniform keypoints\n",
+ " input_keypoints=np.quantile(training_data_df['trestbps'],\n",
+ " np.linspace(0.0, 1.0, num=5)),\n",
+ " dtype=tf.float32,\n",
+ " # Together with quantile keypoints you might want to initialize piecewise\n",
+ " # linear function to have 'equal_slopes' in order for output of layer\n",
+ " # after initialization to preserve original distribution.\n",
+ " kernel_initializer='equal_slopes',\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[3] - 1.0,\n",
+ " # You might consider clamping extreme inputs of the calibrator to output\n",
+ " # bounds.\n",
+ " clamp_min=True,\n",
+ " clamp_max=True,\n",
+ " monotonicity='increasing')\n",
+ "combined_calibrators.append(calibrator)\n",
+ "\n",
+ "# ############### chol ###############\n",
+ "calibrator = tfl.layers.PWLCalibration(\n",
+ " # Explicit input keypoint initialization.\n",
+ " input_keypoints=[126.0, 210.0, 247.0, 286.0, 564.0],\n",
+ " dtype=tf.float32,\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[4] - 1.0,\n",
+ " # Monotonicity of calibrator can be decreasing. Note that corresponding\n",
+ " # lattice dimension must have INCREASING monotonicity regardless of\n",
+ " # monotonicity direction of calibrator.\n",
+ " monotonicity='decreasing',\n",
+ " # Convexity together with decreasing monotonicity result in diminishing\n",
+ " # return constraint.\n",
+ " convexity='convex',\n",
+ " # You can specify list of regularizers. You are not limited to TFL\n",
+ " # regularizrs. Feel free to use any :)\n",
+ " kernel_regularizer=[('laplacian', 0.0, 1e-4),\n",
+ " tf.keras.regularizers.l1_l2(l1=0.001)])\n",
+ "combined_calibrators.append(calibrator)\n",
+ "\n",
+ "# ############### fbs ###############\n",
+ "calibrator = tfl.layers.CategoricalCalibration(\n",
+ " num_buckets=2,\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[5] - 1.0,\n",
+ " # For categorical calibration layer monotonicity is specified for pairs\n",
+ " # of indices of categories. Output for first category in pair will be\n",
+ " # smaller than output for second category.\n",
+ " #\n",
+ " # Don't forget to set monotonicity of corresponding dimension of Lattice\n",
+ " # layer to '1'.\n",
+ " monotonicities=[(0, 1)],\n",
+ " # This initializer is identical to default one('uniform'), but has fixed\n",
+ " # seed in order to simplify experimentation.\n",
+ " kernel_initializer=tf.keras.initializers.RandomUniform(\n",
+ " minval=0.0, maxval=lattice_sizes[5] - 1.0, seed=1))\n",
+ "combined_calibrators.append(calibrator)\n",
+ "\n",
+ "# ############### restecg ###############\n",
+ "calibrator = tfl.layers.CategoricalCalibration(\n",
+ " num_buckets=3,\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[6] - 1.0,\n",
+ " # Categorical monotonicity can be partial order.\n",
+ " monotonicities=[(0, 1), (0, 2)],\n",
+ " # Categorical calibration layer supports standard Keras regularizers.\n",
+ " kernel_regularizer=tf.keras.regularizers.l1_l2(l1=0.001),\n",
+ " kernel_initializer='constant')\n",
+ "combined_calibrators.append(calibrator)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "inyNlSBeQyp7"
+ },
+ "source": [
+ "We then create a lattice layer to nonlinearly fuse the outputs of the calibrators.\n",
+ "\n",
+ "Note that we need to specify the monotonicity of the lattice to be increasing for required dimensions. The composition with the direction of the monotonicity in the calibration will result in the correct end-to-end direction of monotonicity. This includes partial monotonicity of CategoricalCalibration layer."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "DNCc9oBTRo6w"
+ },
+ "outputs": [],
+ "source": [
+ "lattice = tfl.layers.Lattice(\n",
+ " lattice_sizes=lattice_sizes,\n",
+ " monotonicities=[\n",
+ " 'increasing', 'none', 'increasing', 'increasing', 'increasing',\n",
+ " 'increasing', 'increasing'\n",
+ " ],\n",
+ " output_min=0.0,\n",
+ " output_max=1.0)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "T5q2InayRpDr"
+ },
+ "source": [
+ "We can then create a sequential model using the combined calibrators and lattice layers."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "xX6lroYZQy3L"
+ },
+ "outputs": [],
+ "source": [
+ "model = tf.keras.models.Sequential()\n",
+ "model.add(combined_calibrators)\n",
+ "model.add(lattice)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "W3UFxD3fRzIC"
+ },
+ "source": [
+ "Training works the same as any other keras model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "2jz4JvI-RzSj"
+ },
+ "outputs": [],
+ "source": [
+ "features = training_data_df[[\n",
+ " 'age', 'sex', 'cp', 'trestbps', 'chol', 'fbs', 'restecg'\n",
+ "]].values.astype(np.float32)\n",
+ "target = training_data_df[['target']].values.astype(np.float32)\n",
+ "\n",
+ "model.compile(\n",
+ " loss=tf.keras.losses.mean_squared_error,\n",
+ " optimizer=tf.keras.optimizers.Adagrad(learning_rate=LEARNING_RATE))\n",
+ "model.fit(\n",
+ " features,\n",
+ " target,\n",
+ " batch_size=BATCH_SIZE,\n",
+ " epochs=NUM_EPOCHS,\n",
+ " validation_split=0.2,\n",
+ " shuffle=False,\n",
+ " verbose=0)\n",
+ "\n",
+ "model.evaluate(features, target)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "RTHoW_5lxwT5"
+ },
+ "source": [
+ "## Functional Keras Model\n",
+ "\n",
+ "This example uses a functional API for Keras model construction.\n",
+ "\n",
+ "As mentioned in the previous section, lattice layers expect `input[i]` to be within `[0, lattice_sizes[i] - 1.0]`, so we need to define the lattice sizes ahead of the calibration layers so we can properly specify output range of the calibration layers."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "gJjUYvBuW1qE"
+ },
+ "outputs": [],
+ "source": [
+ "# We are going to have 2-d embedding as one of lattice inputs.\n",
+ "lattice_sizes = [3, 2, 2, 3, 3, 2, 2]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "Z03qY5MYW1yT"
+ },
+ "source": [
+ "For each feature, we need to create an input layer followed by a calibration layer. For numeric features we use `tfl.layers.PWLCalibration` and for categorical features we use `tfl.layers.CategoricalCalibration`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "DCIUz8apzs0l"
+ },
+ "outputs": [],
+ "source": [
+ "model_inputs = []\n",
+ "lattice_inputs = []\n",
+ "# ############### age ###############\n",
+ "age_input = tf.keras.layers.Input(shape=[1], name='age')\n",
+ "model_inputs.append(age_input)\n",
+ "age_calibrator = tfl.layers.PWLCalibration(\n",
+ " # Every PWLCalibration layer must have keypoints of piecewise linear\n",
+ " # function specified. Easiest way to specify them is to uniformly cover\n",
+ " # entire input range by using numpy.linspace().\n",
+ " input_keypoints=np.linspace(\n",
+ " training_data_df['age'].min(), training_data_df['age'].max(), num=5),\n",
+ " # You need to ensure that input keypoints have same dtype as layer input.\n",
+ " # You can do it by setting dtype here or by providing keypoints in such\n",
+ " # format which will be converted to deisred tf.dtype by default.\n",
+ " dtype=tf.float32,\n",
+ " # Output range must correspond to expected lattice input range.\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[0] - 1.0,\n",
+ " monotonicity='increasing',\n",
+ " name='age_calib',\n",
+ ")(\n",
+ " age_input)\n",
+ "lattice_inputs.append(age_calibrator)\n",
+ "\n",
+ "# ############### sex ###############\n",
+ "# For boolean features simply specify CategoricalCalibration layer with 2\n",
+ "# buckets.\n",
+ "sex_input = tf.keras.layers.Input(shape=[1], name='sex')\n",
+ "model_inputs.append(sex_input)\n",
+ "sex_calibrator = tfl.layers.CategoricalCalibration(\n",
+ " num_buckets=2,\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[1] - 1.0,\n",
+ " # Initializes all outputs to (output_min + output_max) / 2.0.\n",
+ " kernel_initializer='constant',\n",
+ " name='sex_calib',\n",
+ ")(\n",
+ " sex_input)\n",
+ "lattice_inputs.append(sex_calibrator)\n",
+ "\n",
+ "# ############### cp ###############\n",
+ "cp_input = tf.keras.layers.Input(shape=[1], name='cp')\n",
+ "model_inputs.append(cp_input)\n",
+ "cp_calibrator = tfl.layers.PWLCalibration(\n",
+ " # Here instead of specifying dtype of layer we convert keypoints into\n",
+ " # np.float32.\n",
+ " input_keypoints=np.linspace(1, 4, num=4, dtype=np.float32),\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[2] - 1.0,\n",
+ " monotonicity='increasing',\n",
+ " # You can specify TFL regularizers as tuple ('regularizer name', l1, l2).\n",
+ " kernel_regularizer=('hessian', 0.0, 1e-4),\n",
+ " name='cp_calib',\n",
+ ")(\n",
+ " cp_input)\n",
+ "lattice_inputs.append(cp_calibrator)\n",
+ "\n",
+ "# ############### trestbps ###############\n",
+ "trestbps_input = tf.keras.layers.Input(shape=[1], name='trestbps')\n",
+ "model_inputs.append(trestbps_input)\n",
+ "trestbps_calibrator = tfl.layers.PWLCalibration(\n",
+ " # Alternatively, you might want to use quantiles as keypoints instead of\n",
+ " # uniform keypoints\n",
+ " input_keypoints=np.quantile(training_data_df['trestbps'],\n",
+ " np.linspace(0.0, 1.0, num=5)),\n",
+ " dtype=tf.float32,\n",
+ " # Together with quantile keypoints you might want to initialize piecewise\n",
+ " # linear function to have 'equal_slopes' in order for output of layer\n",
+ " # after initialization to preserve original distribution.\n",
+ " kernel_initializer='equal_slopes',\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[3] - 1.0,\n",
+ " # You might consider clamping extreme inputs of the calibrator to output\n",
+ " # bounds.\n",
+ " clamp_min=True,\n",
+ " clamp_max=True,\n",
+ " monotonicity='increasing',\n",
+ " name='trestbps_calib',\n",
+ ")(\n",
+ " trestbps_input)\n",
+ "lattice_inputs.append(trestbps_calibrator)\n",
+ "\n",
+ "# ############### chol ###############\n",
+ "chol_input = tf.keras.layers.Input(shape=[1], name='chol')\n",
+ "model_inputs.append(chol_input)\n",
+ "chol_calibrator = tfl.layers.PWLCalibration(\n",
+ " # Explicit input keypoint initialization.\n",
+ " input_keypoints=[126.0, 210.0, 247.0, 286.0, 564.0],\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[4] - 1.0,\n",
+ " # Monotonicity of calibrator can be decreasing. Note that corresponding\n",
+ " # lattice dimension must have INCREASING monotonicity regardless of\n",
+ " # monotonicity direction of calibrator.\n",
+ " monotonicity='decreasing',\n",
+ " # Convexity together with decreasing monotonicity result in diminishing\n",
+ " # return constraint.\n",
+ " convexity='convex',\n",
+ " # You can specify list of regularizers. You are not limited to TFL\n",
+ " # regularizrs. Feel free to use any :)\n",
+ " kernel_regularizer=[('laplacian', 0.0, 1e-4),\n",
+ " tf.keras.regularizers.l1_l2(l1=0.001)],\n",
+ " name='chol_calib',\n",
+ ")(\n",
+ " chol_input)\n",
+ "lattice_inputs.append(chol_calibrator)\n",
+ "\n",
+ "# ############### fbs ###############\n",
+ "fbs_input = tf.keras.layers.Input(shape=[1], name='fbs')\n",
+ "model_inputs.append(fbs_input)\n",
+ "fbs_calibrator = tfl.layers.CategoricalCalibration(\n",
+ " num_buckets=2,\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[5] - 1.0,\n",
+ " # For categorical calibration layer monotonicity is specified for pairs\n",
+ " # of indices of categories. Output for first category in pair will be\n",
+ " # smaller than output for second category.\n",
+ " #\n",
+ " # Don't forget to set monotonicity of corresponding dimension of Lattice\n",
+ " # layer to '1'.\n",
+ " monotonicities=[(0, 1)],\n",
+ " # This initializer is identical to default one ('uniform'), but has fixed\n",
+ " # seed in order to simplify experimentation.\n",
+ " kernel_initializer=tf.keras.initializers.RandomUniform(\n",
+ " minval=0.0, maxval=lattice_sizes[5] - 1.0, seed=1),\n",
+ " name='fbs_calib',\n",
+ ")(\n",
+ " fbs_input)\n",
+ "lattice_inputs.append(fbs_calibrator)\n",
+ "\n",
+ "# ############### restecg ###############\n",
+ "restecg_input = tf.keras.layers.Input(shape=[1], name='restecg')\n",
+ "model_inputs.append(restecg_input)\n",
+ "restecg_calibrator = tfl.layers.CategoricalCalibration(\n",
+ " num_buckets=3,\n",
+ " output_min=0.0,\n",
+ " output_max=lattice_sizes[6] - 1.0,\n",
+ " # Categorical monotonicity can be partial order.\n",
+ " monotonicities=[(0, 1), (0, 2)],\n",
+ " # Categorical calibration layer supports standard Keras regularizers.\n",
+ " kernel_regularizer=tf.keras.regularizers.l1_l2(l1=0.001),\n",
+ " kernel_initializer='constant',\n",
+ " name='restecg_calib',\n",
+ ")(\n",
+ " restecg_input)\n",
+ "lattice_inputs.append(restecg_calibrator)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "Fr0k8La_YgQG"
+ },
+ "source": [
+ "We then create a lattice layer to nonlinearly fuse the outputs of the calibrators.\n",
+ "\n",
+ "Note that we need to specify the monotonicity of the lattice to be increasing for required dimensions. The composition with the direction of the monotonicity in the calibration will result in the correct end-to-end direction of monotonicity. This includes partial monotonicity of `tfl.layers.CategoricalCalibration` layer."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "X15RE0NybNbU"
+ },
+ "outputs": [],
+ "source": [
+ "lattice = tfl.layers.Lattice(\n",
+ " lattice_sizes=lattice_sizes,\n",
+ " monotonicities=[\n",
+ " 'increasing', 'none', 'increasing', 'increasing', 'increasing',\n",
+ " 'increasing', 'increasing'\n",
+ " ],\n",
+ " output_min=0.0,\n",
+ " output_max=1.0,\n",
+ " name='lattice',\n",
+ ")(\n",
+ " lattice_inputs)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "31VzsnMCA9dh"
+ },
+ "source": [
+ "To add more flexibility to the model, we add an output calibration layer."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "efCP3Yx2A9n7"
+ },
+ "outputs": [],
+ "source": [
+ "model_output = tfl.layers.PWLCalibration(\n",
+ " input_keypoints=np.linspace(0.0, 1.0, 5),\n",
+ " name='output_calib',\n",
+ ")(\n",
+ " lattice)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "1SURnNl8bNgw"
+ },
+ "source": [
+ "We can now create a model using the inputs and outputs."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "7gY-VXuYbZLa"
+ },
+ "outputs": [],
+ "source": [
+ "model = tf.keras.models.Model(\n",
+ " inputs=model_inputs,\n",
+ " outputs=model_output)\n",
+ "tf.keras.utils.plot_model(model, rankdir='LR')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "tvFJTs94bZXK"
+ },
+ "source": [
+ "Training works the same as any other keras model. Note that, with our setup, input features are passed as separate tensors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "vMQTGbFAYgYS"
+ },
+ "outputs": [],
+ "source": [
+ "feature_names = ['age', 'sex', 'cp', 'trestbps', 'chol', 'fbs', 'restecg']\n",
+ "features = np.split(\n",
+ " training_data_df[feature_names].values.astype(np.float32),\n",
+ " indices_or_sections=len(feature_names),\n",
+ " axis=1)\n",
+ "target = training_data_df[['target']].values.astype(np.float32)\n",
+ "\n",
+ "model.compile(\n",
+ " loss=tf.keras.losses.mean_squared_error,\n",
+ " optimizer=tf.keras.optimizers.Adagrad(LEARNING_RATE))\n",
+ "model.fit(\n",
+ " features,\n",
+ " target,\n",
+ " batch_size=BATCH_SIZE,\n",
+ " epochs=NUM_EPOCHS,\n",
+ " validation_split=0.2,\n",
+ " shuffle=False,\n",
+ " verbose=0)\n",
+ "\n",
+ "model.evaluate(features, target)"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [],
+ "name": "tfl_keras_layers.ipynb",
+ "private_outputs": true,
+ "provenance": [
+ {
+ "file_id": "1ov3qXThgltj77os4ULx7nI63f3oM0vsc",
+ "timestamp": 1579561232062
+ },
+ {
+ "file_id": "1YQhpyfKAW4Gz49gDFMJtVSpAM-Zi12h9",
+ "timestamp": 1579117071304
+ }
+ ],
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/docs/tutorials/shape_constraints.ipynb b/docs/tutorials/shape_constraints.ipynb
new file mode 100644
index 0000000..0065400
--- /dev/null
+++ b/docs/tutorials/shape_constraints.ipynb
@@ -0,0 +1,1262 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "7765UFHoyGx6"
+ },
+ "source": [
+ "##### Copyright 2020 The TensorFlow Authors."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "form",
+ "colab": {},
+ "colab_type": "code",
+ "id": "KsOkK8O69PyT"
+ },
+ "outputs": [],
+ "source": [
+ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+ "# you may not use this file except in compliance with the License.\n",
+ "# You may obtain a copy of the License at\n",
+ "#\n",
+ "# https://www.apache.org/licenses/LICENSE-2.0\n",
+ "#\n",
+ "# Unless required by applicable law or agreed to in writing, software\n",
+ "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+ "# See the License for the specific language governing permissions and\n",
+ "# limitations under the License."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "RKQpW0JqQQmY"
+ },
+ "source": [
+ "# Shape Constraints with Tensorflow Lattice\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "r61fkA2i9Y3_"
+ },
+ "source": [
+ "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/lattice/tutorials/shape_constraints\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/lattice/blob/master/docs/tutorials/shape_constraints.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/lattice/blob/master/docs/tutorials/shape_constraints.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ " \u003ctd\u003e\n",
+ " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/lattice/tutorials/shape_constraints.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
+ " \u003c/td\u003e\n",
+ "\u003c/table\u003e"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "2plcL3iTVjsp"
+ },
+ "source": [
+ "## Overview\n",
+ "\n",
+ "In this colab we will fit a TensorFlow Lattice (TFL) canned classifier on a handcrafted restaurant review rating dataset and experiment with various TFL regularizers and shape constraints. Before proceeding, make sure your runtime has all required packages installed (as imported in the code cells below)."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "x769lI12IZXB"
+ },
+ "source": [
+ "## Setup"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "fbBVAR6UeRN5"
+ },
+ "source": [
+ "Installing TF Lattice package:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "bpXjJKpSd3j4"
+ },
+ "outputs": [],
+ "source": [
+ "#@test {\"skip\": true}\n",
+ "!pip install tensorflow-lattice"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "jSVl9SHTeSGX"
+ },
+ "source": [
+ "Importing required packages:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "both",
+ "colab": {},
+ "colab_type": "code",
+ "id": "iY6awAl058TV"
+ },
+ "outputs": [],
+ "source": [
+ "from __future__ import absolute_import, division, print_function, unicode_literals\n",
+ "!pip install tensorflow-lattice\n",
+ "\n",
+ "try:\n",
+ " # %tensorflow_version only exists in Colab.\n",
+ " %tensorflow_version 2.x\n",
+ "except Exception:\n",
+ " pass\n",
+ "import tensorflow as tf\n",
+ "\n",
+ "from IPython.core.pylabtools import figsize\n",
+ "import itertools\n",
+ "import matplotlib\n",
+ "from matplotlib import pyplot as plt\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import tensorflow_lattice as tfl"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "7TmBk_IGgJF0"
+ },
+ "source": [
+ "Default values used in this guide:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "kQHPyPsPUF92"
+ },
+ "outputs": [],
+ "source": [
+ "NUM_EPOCHS = 500\n",
+ "BATCH_SIZE = 64\n",
+ "LEARNING_RATE=0.001"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "FjR7D8Ag3z0d"
+ },
+ "source": [
+ "## Training Dataset for Ranking Restaurants"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "a1YetzbdFOij"
+ },
+ "source": [
+ "Imagine a simplified scenario where we want to determine whether or not users will click on a restaurant search. The task is to predict the clickthrough rate (CTR) given input features:\n",
+ "- Average rating (`avg_rating`): a numeric (float) feature with values in the range [1,5].\n",
+ "- Number of reviews (`num_reviews`): a positive numeric (integer) feature with values capped at 200, and used as a measure of trendiness.\n",
+ "- Dollar rating (`dollar_rating`): a categorical feature with string values in the set {\"D\", \"DD\", \"DDD\", \"DDDD\"}.\n",
+ "\n",
+ "Here we create a synthetic dataset where the true CTR is given by the formula:\n",
+ "$$\n",
+ "CTR = \\frac{1}{1 + exp\\{\\mbox{b(dollar_rating)}-\\mbox{avg_rating}\\times log(\\mbox{num_reviews}) /4 \\}}, \n",
+ "$$\n",
+ "where $b(\\cdot)$ translates each `dollar_rating` to a baseline value:\n",
+ "$$\n",
+ "\\mbox{D}\\to 3,\\ \\mbox{DD}\\to 2,\\ \\mbox{DDD}\\to 4,\\ \\mbox{DDDD}\\to 4.5. \n",
+ "$$\n",
+ "\n",
+ "This formula reflects typical user patterns. e.g. given everything else fixed, \"\\\\$\\\\$\" restaurants will receive more clicks than \"\\\\$\", followed by \"\\\\$\\\\$\\\\$\" and \"\\\\$\\\\$\\\\$\\\\$\". "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "mKovnyv1jATw"
+ },
+ "outputs": [],
+ "source": [
+ "def click_through_rate(avg_ratings, num_reviews, dollar_ratings):\n",
+ " dollar_rating_baseline = {\"D\": 3, \"DD\": 2, \"DDD\": 4, \"DDDD\": 4.5}\n",
+ " return 1 / (1 + np.exp(\n",
+ " np.array([dollar_rating_baseline[d] for d in dollar_ratings]) -\n",
+ " avg_ratings * np.log1p(num_reviews) / 4))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "BPlgRdt6jAbP"
+ },
+ "source": [
+ "Let's take a look at the contour plots of this CTR function."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "KC5qX_XKmc7g"
+ },
+ "outputs": [],
+ "source": [
+ "def color_bar():\n",
+ " bar = matplotlib.cm.ScalarMappable(\n",
+ " norm=matplotlib.colors.Normalize(0, 1, True),\n",
+ " cmap=\"viridis\",\n",
+ " )\n",
+ " bar.set_array([0, 1])\n",
+ " return bar\n",
+ "\n",
+ "\n",
+ "def plot_fns(fns, split_by_dollar=False, res=25):\n",
+ " \"\"\"Generates contour plots for a list of (name, fn) functions.\"\"\"\n",
+ " num_reviews, avg_ratings = np.meshgrid(\n",
+ " np.linspace(0, 200, num=res),\n",
+ " np.linspace(1, 5, num=res),\n",
+ " )\n",
+ " if split_by_dollar:\n",
+ " dollar_rating_splits = [\"D\", \"DD\", \"DDD\", \"DDDD\"]\n",
+ " else:\n",
+ " dollar_rating_splits = [None]\n",
+ " if len(fns) == 1:\n",
+ " fig, axes = plt.subplots(2, 2, sharey=True, tight_layout=False)\n",
+ " else:\n",
+ " fig, axes = plt.subplots(\n",
+ " len(dollar_rating_splits), len(fns), sharey=True, tight_layout=False)\n",
+ " axes = axes.flatten()\n",
+ " axes_index = 0\n",
+ " for dollar_rating_split in dollar_rating_splits:\n",
+ " for title, fn in fns:\n",
+ " if dollar_rating_split is not None:\n",
+ " dollar_ratings = np.repeat(dollar_rating_split, res**2)\n",
+ " values = fn(avg_ratings.flatten(), num_reviews.flatten(),\n",
+ " dollar_ratings)\n",
+ " title = \"{}: dollar_rating={}\".format(title, dollar_rating_split)\n",
+ " else:\n",
+ " values = fn(avg_ratings.flatten(), num_reviews.flatten())\n",
+ " subplot = axes[axes_index]\n",
+ " axes_index += 1\n",
+ " subplot.contourf(\n",
+ " avg_ratings,\n",
+ " num_reviews,\n",
+ " np.reshape(values, (res, res)),\n",
+ " vmin=0,\n",
+ " vmax=1)\n",
+ " subplot.title.set_text(title)\n",
+ " subplot.set(xlabel=\"Average Rating\")\n",
+ " subplot.set(ylabel=\"Number of Reviews\")\n",
+ " subplot.set(xlim=(1, 5))\n",
+ "\n",
+ " _ = fig.colorbar(color_bar(), cax=fig.add_axes([0.95, 0.2, 0.01, 0.6]))\n",
+ "\n",
+ "\n",
+ "figsize(11, 11)\n",
+ "plot_fns([(\"CTR\", click_through_rate)], split_by_dollar=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "Ol91olp3muNN"
+ },
+ "source": [
+ "### Preparing Data\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "H8BOshZS9xwn"
+ },
+ "source": [
+ "We start by generating a simulated dataset of restaurants and their features."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "MhqcOPdTT_wj"
+ },
+ "outputs": [],
+ "source": [
+ "def sample_restaurants(n):\n",
+ " avg_ratings = np.random.uniform(1.0, 5.0, n)\n",
+ " num_reviews = np.round(np.exp(np.random.uniform(0.0, np.log(200), n)))\n",
+ " dollar_ratings = np.random.choice([\"D\", \"DD\", \"DDD\", \"DDDD\"], n)\n",
+ " ctr_labels = click_through_rate(avg_ratings, num_reviews, dollar_ratings)\n",
+ " return avg_ratings, num_reviews, dollar_ratings, ctr_labels\n",
+ "\n",
+ "\n",
+ "np.random.seed(42)\n",
+ "avg_ratings, num_reviews, dollar_ratings, ctr_labels = sample_restaurants(2000)\n",
+ "\n",
+ "figsize(5, 5)\n",
+ "fig, axs = plt.subplots(1, 1, sharey=False, tight_layout=False)\n",
+ "for rating, marker in [(\"D\", \"o\"), (\"DD\", \"^\"), (\"DDD\", \"+\"), (\"DDDD\", \"x\")]:\n",
+ " plt.scatter(\n",
+ " x=avg_ratings[np.where(dollar_ratings == rating)],\n",
+ " y=num_reviews[np.where(dollar_ratings == rating)],\n",
+ " c=ctr_labels[np.where(dollar_ratings == rating)],\n",
+ " vmin=0,\n",
+ " vmax=1,\n",
+ " marker=marker,\n",
+ " label=rating)\n",
+ "plt.xlabel(\"Average Rating\")\n",
+ "plt.ylabel(\"Number of Reviews\")\n",
+ "plt.legend()\n",
+ "plt.xlim((1, 5))\n",
+ "plt.title(\"Distribution of restaurants\")\n",
+ "_ = fig.colorbar(color_bar(), cax=fig.add_axes([0.95, 0.2, 0.01, 0.6]))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "tRetsfLv_JSR"
+ },
+ "source": [
+ "Let's produce the training, validation and testing datasets. When a restaurant is viewed in the search results, we can record user's engagement (click or no click) as a sample point. \n",
+ "\n",
+ "In practice, users often do not go through all search results. This means that users will likely only see restaurants already considered \"good\" by the current ranking model in use. As a result, \"good\" restaurants are more frequently impressed and over-represented in the training datasets.\n",
+ "\n",
+ "When the model is used for ranking, it is often evaluated on all relevant results with a more uniform distribution. As a result, it may act unexpectedly at evaluation time for cases that were are under-represented in the training dataset. When using more features, the training dataset can have large gaps in \"bad\" parts of the feature space.\n",
+ "\n",
+ "A flexible and complicated model might fail in this case due to overfitting the over-represented data points and thus lacking generalizability. We handle this issue by applying domain knowledge to add *shape constraints* that guide the trained model to make reasonable predictions *when it cannot pick them up from the training dataset*.\n",
+ "\n",
+ "In this example, for the testing dataset we intentionally ignore the over-representation to simulate the online setting previously discussed."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "jS6WOtXQ8jwX"
+ },
+ "outputs": [],
+ "source": [
+ "def sample_dataset(n, testing_set):\n",
+ " (avg_ratings, num_reviews, dollar_ratings, ctr_labels) = sample_restaurants(n)\n",
+ " if testing_set:\n",
+ " # Testing has a more uniform distribution over all restaurants.\n",
+ " num_views = np.random.poisson(lam=3, size=n)\n",
+ " else:\n",
+ " # Training/validation datasets have more views on popular restaurants.\n",
+ " num_views = np.random.poisson(lam=ctr_labels * num_reviews / 50.0, size=n)\n",
+ "\n",
+ " return pd.DataFrame({\n",
+ " \"avg_rating\": np.repeat(avg_ratings, num_views),\n",
+ " \"num_reviews\": np.repeat(num_reviews, num_views),\n",
+ " \"dollar_rating\": np.repeat(dollar_ratings, num_views),\n",
+ " \"clicked\": np.random.binomial(n=1, p=np.repeat(ctr_labels, num_views))\n",
+ " })\n",
+ "\n",
+ "\n",
+ "# Generate datasets.\n",
+ "np.random.seed(42)\n",
+ "data_train = sample_dataset(2000, testing_set=False)\n",
+ "data_val = sample_dataset(1000, testing_set=False)\n",
+ "data_test = sample_dataset(1000, testing_set=True)\n",
+ "\n",
+ "# Plotting dataset densities.\n",
+ "figsize(12, 5)\n",
+ "fig, axs = plt.subplots(1, 2, sharey=False, tight_layout=False)\n",
+ "for ax, data, title in [(axs[0], data_train, \"training\"),\n",
+ " (axs[1], data_test, \"testing\")]:\n",
+ " _, _, _, density = ax.hist2d(\n",
+ " x=data[\"avg_rating\"],\n",
+ " y=data[\"num_reviews\"],\n",
+ " bins=(np.linspace(1, 5, num=21), np.linspace(0, 200, num=21)),\n",
+ " normed=True,\n",
+ " cmap=\"Blues\",\n",
+ " )\n",
+ " ax.set(xlim=(1, 5))\n",
+ " ax.set(ylim=(0, 200))\n",
+ " ax.set(xlabel=\"Average Rating\")\n",
+ " ax.set(ylabel=\"Number of Reviews\")\n",
+ " ax.title.set_text(\"Density of {} examples\".format(title))\n",
+ " _ = fig.colorbar(density, ax=ax)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "4fVyLgpCT1nW"
+ },
+ "source": [
+ "Defining input_fns used for training and evaluation:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "DYzRTRR2GKoS"
+ },
+ "outputs": [],
+ "source": [
+ "train_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(\n",
+ " x=data_train,\n",
+ " y=data_train[\"clicked\"],\n",
+ " batch_size=BATCH_SIZE,\n",
+ " num_epochs=NUM_EPOCHS,\n",
+ " shuffle=False,\n",
+ ")\n",
+ "\n",
+ "# feature_analysis_input_fn is used for TF Lattice estimators.\n",
+ "feature_analysis_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(\n",
+ " x=data_train,\n",
+ " y=data_train[\"clicked\"],\n",
+ " batch_size=BATCH_SIZE,\n",
+ " num_epochs=1,\n",
+ " shuffle=False,\n",
+ ")\n",
+ "\n",
+ "val_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(\n",
+ " x=data_val,\n",
+ " y=data_val[\"clicked\"],\n",
+ " batch_size=BATCH_SIZE,\n",
+ " num_epochs=1,\n",
+ " shuffle=False,\n",
+ ")\n",
+ "\n",
+ "test_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(\n",
+ " x=data_test,\n",
+ " y=data_test[\"clicked\"],\n",
+ " batch_size=BATCH_SIZE,\n",
+ " num_epochs=1,\n",
+ " shuffle=False,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "qoTrw3FZqvPK"
+ },
+ "source": [
+ "## Fitting Gradient Boosted Trees"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "ZklNowexE3wB"
+ },
+ "source": [
+ "Let's start off with only two features: `avg_rating` and `num_reviews`.\n",
+ "\n",
+ "We create a few auxillary functions for plotting and calculating validation and test metrics."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "SX6rARJWURWl"
+ },
+ "outputs": [],
+ "source": [
+ "def analyze_two_d_estimator(estimator, name):\n",
+ " # Extract validation metrics.\n",
+ " metric = estimator.evaluate(input_fn=val_input_fn)\n",
+ " print(\"Validation AUC: {}\".format(metric[\"auc\"]))\n",
+ " metric = estimator.evaluate(input_fn=test_input_fn)\n",
+ " print(\"Testing AUC: {}\".format(metric[\"auc\"]))\n",
+ "\n",
+ " def two_d_pred(avg_ratings, num_reviews):\n",
+ " results = estimator.predict(\n",
+ " tf.compat.v1.estimator.inputs.pandas_input_fn(\n",
+ " x=pd.DataFrame({\n",
+ " \"avg_rating\": avg_ratings,\n",
+ " \"num_reviews\": num_reviews,\n",
+ " }),\n",
+ " shuffle=False,\n",
+ " ))\n",
+ " return [x[\"logistic\"][0] for x in results]\n",
+ "\n",
+ " def two_d_click_through_rate(avg_ratings, num_reviews):\n",
+ " return np.mean([\n",
+ " click_through_rate(avg_ratings, num_reviews,\n",
+ " np.repeat(d, len(avg_ratings)))\n",
+ " for d in [\"D\", \"DD\", \"DDD\", \"DDDD\"]\n",
+ " ],\n",
+ " axis=0)\n",
+ "\n",
+ " figsize(11, 5)\n",
+ " plot_fns([(\"{} Estimated CTR\".format(name), two_d_pred),\n",
+ " (\"CTR\", two_d_click_through_rate)],\n",
+ " split_by_dollar=False)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "JVef4f8yUUbs"
+ },
+ "source": [
+ "We can fit TensorFlow gradient boosted decision trees on the dataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "DnPYlRAo2mnQ"
+ },
+ "outputs": [],
+ "source": [
+ "feature_columns = [\n",
+ " tf.feature_column.numeric_column(\"num_reviews\"),\n",
+ " tf.feature_column.numeric_column(\"avg_rating\"),\n",
+ "]\n",
+ "gbt_estimator = tf.estimator.BoostedTreesClassifier(\n",
+ " feature_columns=feature_columns,\n",
+ " # Hyper-params optimized on validation set.\n",
+ " n_batches_per_layer=100,\n",
+ " max_depth=2,\n",
+ " n_trees=100,\n",
+ " config=tf.estimator.RunConfig(tf_random_seed=42),\n",
+ ")\n",
+ "gbt_estimator.train(input_fn=train_input_fn)\n",
+ "analyze_two_d_estimator(gbt_estimator, \"GBT\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "nYZtd6YvsNdn"
+ },
+ "source": [
+ "Even though the model has captured the general shape of the true CTR, it produces a counter-intuitive contour plot of the estimated (predicted) CTR: certain spots on the contour surface can see an increased estimated CTR if we move either downwards or leftwards, meaning that one would be more likely to click on the restaurants at those spots if\n",
+ "- their review numbers decreased, i.e. they were less trendy, or\n",
+ "- their average ratings dropped, i.e. they served worse food.\n",
+ "\n",
+ "A couple of reasons may be accountable:\n",
+ "- We are missing `dollar_rating`, an important feature.\n",
+ "- The traing dataset has \"holes\": areas where there are not enough sample points.\n",
+ "- Noise signal ratio is high.\n",
+ "- Applied ML model (boosted trees) is too flexibile and is easy to overfit.\n",
+ "\n",
+ "The remedy could be simple: we enforce the shape constraint that the model must estimate CTR values monotonically increasing with respect to both the average rating and the number of reviews. We will later see how to implement this in TFL.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "Uf7WqGooFiEp"
+ },
+ "source": [
+ "## Fitting a DNN"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "_s2aT3x0E_tF"
+ },
+ "source": [
+ "We can repeat the same steps with a DNN classifier. Similar patterns can also be observed (you can rerun the cell if not so): not enough sample points are in the area where `num_reviews` is small, and the DNN extrapolation in this area works poorly. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "gFUeG6kLDNhO"
+ },
+ "outputs": [],
+ "source": [
+ "feature_columns = [\n",
+ " tf.feature_column.numeric_column(\"num_reviews\"),\n",
+ " tf.feature_column.numeric_column(\"avg_rating\"),\n",
+ "]\n",
+ "dnn_estimator = tf.estimator.DNNClassifier(\n",
+ " feature_columns=feature_columns,\n",
+ " # Hyper-params optimized on validation set.\n",
+ " hidden_units=[16, 8, 8],\n",
+ " optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n",
+ " config=tf.estimator.RunConfig(tf_random_seed=42),\n",
+ ")\n",
+ "dnn_estimator.train(input_fn=train_input_fn)\n",
+ "analyze_two_d_estimator(dnn_estimator, \"DNN\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "0Avkw-okw7JL"
+ },
+ "source": [
+ "## Shape Constraints"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "3ExyethCFBrP"
+ },
+ "source": [
+ "TensorFlow Lattice (TFL) is laser-focusing on installing shape constraints to safeguard model behavior. These shape constraints are fulfilled by two key concepts of TFL: \n",
+ "- *calibrator*: a piece-wise linear function, and\n",
+ "- *lattice*: a multi-dimentional lookup table. \n",
+ "\n",
+ "Their details can be found in [our JMLR paper](http://jmlr.org/papers/volume17/15-243/15-243.pdf). \n",
+ "\n",
+ "The most straightforward way to use TFL is through the premade TFL canned estimators. In this colab we will configure a TFL canned classifier. Similar to a TensorFlow estimator, training a TFL canned estimator requires several components:\n",
+ "- feature columns: definition of model features.\n",
+ "- feature configs: definition of TFL specific feature specs and shape constraints.\n",
+ "- model config: configuration of TFL canned estimator specs.\n",
+ "- feature analysis input fn: a TF input fn passing data for TFL initialization.\n",
+ "- (train) input fn: a TF input fn passing data for model training."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "anyCM4sCpOSo"
+ },
+ "source": [
+ "### Monotonicity\n",
+ "We first address our monotonicity concerns by adding monotonicity shape constraints to both features involved. \n",
+ "\n",
+ "To instruct TFL to install shape constraints, we decide which features are involved and specify the constraints for any feature in its feature config. The following code shows how we can require the output to be monotonically increasing with respect to both `num_reviews` and `avg_rating`.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "FCm1lOjmwur_"
+ },
+ "outputs": [],
+ "source": [
+ "feature_columns = [\n",
+ " tf.feature_column.numeric_column(\"num_reviews\"),\n",
+ " tf.feature_column.numeric_column(\"avg_rating\"),\n",
+ "]\n",
+ "model_config = tfl.configs.CalibratedLatticeConfig(\n",
+ " feature_configs=[\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"num_reviews\",\n",
+ " lattice_size=2,\n",
+ " monotonicity=\"increasing\",\n",
+ " pwl_calibration_num_keypoints=20,\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"avg_rating\",\n",
+ " lattice_size=2,\n",
+ " monotonicity=\"increasing\",\n",
+ " pwl_calibration_num_keypoints=20,\n",
+ " )\n",
+ " ])\n",
+ "tfl_estimator = tfl.estimators.CannedClassifier(\n",
+ " feature_columns=feature_columns,\n",
+ " model_config=model_config,\n",
+ " feature_analysis_input_fn=feature_analysis_input_fn,\n",
+ " optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n",
+ " config=tf.estimator.RunConfig(tf_random_seed=42),\n",
+ ")\n",
+ "tfl_estimator.train(input_fn=train_input_fn)\n",
+ "analyze_two_d_estimator(tfl_estimator, \"TF Lattice\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "ubNRBCWW5wQ9"
+ },
+ "source": [
+ "This canned classifier we built behaves in the follow manner:\n",
+ "- It first applies a *calibrator* (a piece-wise linear function) to each feature to map the feature values onto [0,1].\n",
+ "- It then joins the calibrated feature values using a *lattice* and outputs the prediction.\n",
+ "\n",
+ "We can use `tfl.visualization` to visualize model behavior. In particular, the following plot shows the two trained calibrators included in the canned classifier. \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "both",
+ "colab": {},
+ "colab_type": "code",
+ "id": "C0py9Q6OBRBE"
+ },
+ "outputs": [],
+ "source": [
+ "def save_and_visualize_lattice(tfl_estimator):\n",
+ " saved_model_path = tfl_estimator.export_saved_model(\n",
+ " \"/tmp/TensorFlow_Lattice_101/\",\n",
+ " tf.estimator.export.build_parsing_serving_input_receiver_fn(\n",
+ " feature_spec=tf.feature_column.make_parse_example_spec(\n",
+ " feature_columns)))\n",
+ " model_graph = tfl.estimators.get_model_graph(saved_model_path)\n",
+ " figsize(8, 8)\n",
+ " tfl.visualization.draw_model_graph(model_graph)\n",
+ " return model_graph\n",
+ "\n",
+ "_ = save_and_visualize_lattice(tfl_estimator)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "7vZ5fShXs504"
+ },
+ "source": [
+ "Now the contour plot gets cleaner and is showing what makes sense: estimated CTR will go up as long as the average rating increases or the number of reviews increases. Notice that the calibrators are monotonic."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "RfniRZCHIvfK"
+ },
+ "source": [
+ "### Diminishing Returns\n",
+ "[Diminishing returns](https://en.wikipedia.org/wiki/Diminishing_returns) means that the marginal gain of increasing certain a feature value will decrease as we increase the value. In our case we expect that the `num_reviews` feature follows this pattern, so we can configure its calibrator accordingly. Notice that we can decompose diminishing returns into two sufficient conditions:\n",
+ "\n",
+ "- the calibrator is monotonicially increasing, and\n",
+ "- the calibrator is concave.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "XQrM9BskY-wx"
+ },
+ "outputs": [],
+ "source": [
+ "feature_columns = [\n",
+ " tf.feature_column.numeric_column(\"num_reviews\"),\n",
+ " tf.feature_column.numeric_column(\"avg_rating\"),\n",
+ "]\n",
+ "model_config = tfl.configs.CalibratedLatticeConfig(\n",
+ " feature_configs=[\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"num_reviews\",\n",
+ " lattice_size=2,\n",
+ " monotonicity=\"increasing\",\n",
+ " pwl_calibration_convexity=\"concave\",\n",
+ " pwl_calibration_num_keypoints=20,\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"avg_rating\",\n",
+ " lattice_size=2,\n",
+ " monotonicity=\"increasing\",\n",
+ " pwl_calibration_num_keypoints=20,\n",
+ " )\n",
+ " ])\n",
+ "tfl_estimator = tfl.estimators.CannedClassifier(\n",
+ " feature_columns=feature_columns,\n",
+ " model_config=model_config,\n",
+ " feature_analysis_input_fn=feature_analysis_input_fn,\n",
+ " optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n",
+ " config=tf.estimator.RunConfig(tf_random_seed=42),\n",
+ ")\n",
+ "tfl_estimator.train(input_fn=train_input_fn)\n",
+ "analyze_two_d_estimator(tfl_estimator, \"TF Lattice\")\n",
+ "_ = save_and_visualize_lattice(tfl_estimator)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "J6CP2Ovapiu3"
+ },
+ "source": [
+ "### 2D Shape Constraint: Trust\n",
+ "A 5-star rating for a restaurant with only one or two reviews is likely an unreliable rating (the restaurant might not actually be good), whereas a 4-star rating for a restaurant with hundreds of reviews is much more reliable (the restaurant is likely good in this case). We can see that the number of reviews of a restaurant affects how much trust we place in its average rating. \n",
+ "\n",
+ "We can exercise TFL trust constraints to inform the model that the larger (or smaller) value of one feature indicates more usage of another feature. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "OA14j0erm6TJ"
+ },
+ "outputs": [],
+ "source": [
+ "feature_columns = [\n",
+ " tf.feature_column.numeric_column(\"num_reviews\"),\n",
+ " tf.feature_column.numeric_column(\"avg_rating\"),\n",
+ "]\n",
+ "model_config = tfl.configs.CalibratedLatticeConfig(\n",
+ " feature_configs=[\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"num_reviews\",\n",
+ " lattice_size=2,\n",
+ " monotonicity=\"increasing\",\n",
+ " pwl_calibration_convexity=\"concave\",\n",
+ " pwl_calibration_num_keypoints=20,\n",
+ " # Larger num_reviews indicating more trust in avg_rating.\n",
+ " reflects_trust_in=[\n",
+ " tfl.configs.TrustConfig(\n",
+ " feature_name=\"avg_rating\", trust_type=\"edgeworth\"),\n",
+ " ],\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"avg_rating\",\n",
+ " lattice_size=2,\n",
+ " monotonicity=\"increasing\",\n",
+ " pwl_calibration_num_keypoints=20,\n",
+ " )\n",
+ " ])\n",
+ "tfl_estimator = tfl.estimators.CannedClassifier(\n",
+ " feature_columns=feature_columns,\n",
+ " model_config=model_config,\n",
+ " feature_analysis_input_fn=feature_analysis_input_fn,\n",
+ " optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n",
+ " config=tf.estimator.RunConfig(tf_random_seed=42),\n",
+ ")\n",
+ "tfl_estimator.train(input_fn=train_input_fn)\n",
+ "analyze_two_d_estimator(tfl_estimator, \"TF Lattice\")\n",
+ "model_graph = save_and_visualize_lattice(tfl_estimator)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "puvP9X8XxyRV"
+ },
+ "source": [
+ "The following plot presents the trained lattice lookup result. Due to the trust constraint, we would expect that larger values of calibrated `num_reviews` would enable wider ranges for calibrated `avg_rating` to more significantly move the lattice output. This is confirmed by the plot."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "cellView": "both",
+ "colab": {},
+ "colab_type": "code",
+ "id": "RounEQebxxnA"
+ },
+ "outputs": [],
+ "source": [
+ "lat_mesh_n = 12\n",
+ "lat_mesh_x, lat_mesh_y = tfl.test_utils.two_dim_mesh_grid(\n",
+ " lat_mesh_n**2, 0, 0, 1, 1)\n",
+ "lat_mesh_fn = tfl.test_utils.get_hypercube_interpolation_fn(\n",
+ " model_graph.output_node.weights.flatten())\n",
+ "lat_mesh_z = [\n",
+ " lat_mesh_fn([lat_mesh_x.flatten()[i],\n",
+ " lat_mesh_y.flatten()[i]]) for i in range(lat_mesh_n**2)\n",
+ "]\n",
+ "trust_plt = tfl.visualization.plot_outputs(\n",
+ " (lat_mesh_x, lat_mesh_y),\n",
+ " {\"Lattice Lookup\": lat_mesh_z},\n",
+ " figsize=(6, 6),\n",
+ ")\n",
+ "trust_plt.title(\"Trust\")\n",
+ "trust_plt.xlabel(\"Calibrated avg_rating\")\n",
+ "trust_plt.ylabel(\"Calibrated num_reviews\")\n",
+ "trust_plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "SKe3UHX6pUjw"
+ },
+ "source": [
+ "### Smoothing Calibrators\n",
+ "Let's now take a look at the calibrator of `avg_rating`. Though it is monotonically increasing, the changes in its slopes are somewhat random and hard to interpret. That suggests we might want to consider smoothing this calibrator.\n",
+ "\n",
+ "Here we apply a `wrinkle` regularizer to reduce changes in the curvature. There are also the `laplacian` regularizer to flatten the calibrator and the `hessian` regularizer to make it more linear. \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "qxFHH3hSpWfq"
+ },
+ "outputs": [],
+ "source": [
+ "feature_columns = [\n",
+ " tf.feature_column.numeric_column(\"num_reviews\"),\n",
+ " tf.feature_column.numeric_column(\"avg_rating\"),\n",
+ "]\n",
+ "model_config = tfl.configs.CalibratedLatticeConfig(\n",
+ " feature_configs=[\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"num_reviews\",\n",
+ " lattice_size=2,\n",
+ " monotonicity=\"increasing\",\n",
+ " pwl_calibration_convexity=\"concave\",\n",
+ " pwl_calibration_num_keypoints=20,\n",
+ " regularizer_configs=[\n",
+ " tfl.configs.RegularizerConfig(name=\"calib_wrinkle\", l2=1.0),\n",
+ " ],\n",
+ " reflects_trust_in=[\n",
+ " tfl.configs.TrustConfig(\n",
+ " feature_name=\"avg_rating\", trust_type=\"edgeworth\"),\n",
+ " ],\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"avg_rating\",\n",
+ " lattice_size=2,\n",
+ " monotonicity=\"increasing\",\n",
+ " pwl_calibration_num_keypoints=20,\n",
+ " regularizer_configs=[\n",
+ " tfl.configs.RegularizerConfig(name=\"calib_wrinkle\", l2=1.0),\n",
+ " ],\n",
+ " )\n",
+ " ])\n",
+ "tfl_estimator = tfl.estimators.CannedClassifier(\n",
+ " feature_columns=feature_columns,\n",
+ " model_config=model_config,\n",
+ " feature_analysis_input_fn=feature_analysis_input_fn,\n",
+ " optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n",
+ " config=tf.estimator.RunConfig(tf_random_seed=42),\n",
+ ")\n",
+ "tfl_estimator.train(input_fn=train_input_fn)\n",
+ "analyze_two_d_estimator(tfl_estimator, \"TF Lattice\")\n",
+ "_ = save_and_visualize_lattice(tfl_estimator)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "pSUd6aFlpYz4"
+ },
+ "source": [
+ "### Partial Monotonicity for Categorical Calibration\n",
+ "So far we have been using only two of the numeric features in the model. Here we will add a third feature using a categorical calibration layer. Again we start by setting up helper functions for plotting and metric calculation."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "5tLDKwTmjrLw"
+ },
+ "outputs": [],
+ "source": [
+ "def analyze_three_d_estimator(estimator, name):\n",
+ " # Extract validation metrics.\n",
+ " metric = estimator.evaluate(input_fn=val_input_fn)\n",
+ " print(\"Validation AUC: {}\".format(metric[\"auc\"]))\n",
+ " metric = estimator.evaluate(input_fn=test_input_fn)\n",
+ " print(\"Testing AUC: {}\".format(metric[\"auc\"]))\n",
+ "\n",
+ " def three_d_pred(avg_ratings, num_reviews, dollar_rating):\n",
+ " results = estimator.predict(\n",
+ " tf.compat.v1.estimator.inputs.pandas_input_fn(\n",
+ " x=pd.DataFrame({\n",
+ " \"avg_rating\": avg_ratings,\n",
+ " \"num_reviews\": num_reviews,\n",
+ " \"dollar_rating\": dollar_rating,\n",
+ " }),\n",
+ " shuffle=False,\n",
+ " ))\n",
+ " return [x[\"logistic\"][0] for x in results]\n",
+ "\n",
+ " figsize(11, 22)\n",
+ " plot_fns([(\"{} Estimated CTR\".format(name), three_d_pred),\n",
+ " (\"CTR\", click_through_rate)],\n",
+ " split_by_dollar=True)\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "CnPiqf4rq6kJ"
+ },
+ "source": [
+ "To involve the third feature `dollar_rating`, we should recall that categorical features require a slightly different treatment in TFL: both as a feature column and as a feature config. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "m-w7iGEEpgGt"
+ },
+ "outputs": [],
+ "source": [
+ "feature_columns = [\n",
+ " tf.feature_column.numeric_column(\"num_reviews\"),\n",
+ " tf.feature_column.numeric_column(\"avg_rating\"),\n",
+ " tf.feature_column.categorical_column_with_vocabulary_list(\n",
+ " \"dollar_rating\",\n",
+ " vocabulary_list=[\"D\", \"DD\", \"DDD\", \"DDDD\"],\n",
+ " dtype=tf.string,\n",
+ " default_value=0),\n",
+ "]\n",
+ "model_config = tfl.configs.CalibratedLatticeConfig(\n",
+ " feature_configs=[\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"num_reviews\",\n",
+ " lattice_size=2,\n",
+ " monotonicity=\"increasing\",\n",
+ " pwl_calibration_convexity=\"concave\",\n",
+ " pwl_calibration_num_keypoints=20,\n",
+ " regularizer_configs=[\n",
+ " tfl.configs.RegularizerConfig(name=\"calib_wrinkle\", l2=1.0),\n",
+ " ],\n",
+ " reflects_trust_in=[\n",
+ " tfl.configs.TrustConfig(\n",
+ " feature_name=\"avg_rating\", trust_type=\"edgeworth\"),\n",
+ " ],\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"avg_rating\",\n",
+ " lattice_size=2,\n",
+ " monotonicity=\"increasing\",\n",
+ " pwl_calibration_num_keypoints=20,\n",
+ " regularizer_configs=[\n",
+ " tfl.configs.RegularizerConfig(name=\"calib_wrinkle\", l2=1.0),\n",
+ " ],\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"dollar_rating\",\n",
+ " lattice_size=2,\n",
+ " pwl_calibration_num_keypoints=4,\n",
+ " # Here we only specify one monotonicity:\n",
+ " # `D` resturants has smaller value than `DD` restaurants\n",
+ " monotonicity=[(\"D\", \"DD\")],\n",
+ " ),\n",
+ " ])\n",
+ "tfl_estimator = tfl.estimators.CannedClassifier(\n",
+ " feature_columns=feature_columns,\n",
+ " model_config=model_config,\n",
+ " feature_analysis_input_fn=feature_analysis_input_fn,\n",
+ " optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n",
+ " config=tf.estimator.RunConfig(tf_random_seed=42),\n",
+ ")\n",
+ "tfl_estimator.train(input_fn=train_input_fn)\n",
+ "analyze_three_d_estimator(tfl_estimator, \"TF Lattice\")\n",
+ "_ = save_and_visualize_lattice(tfl_estimator)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "gdIzhYL79_Pp"
+ },
+ "source": [
+ "This categorical calibrator shows the preference of the model output: DD \u003e D \u003e DDD \u003e DDDD, which is consistent with our setup. Notice there is also a column for missing values. Though there is no missing feature in our training and testing data, the model provides us with the best way to treat the missing value should it happen during downstream model serving.\n",
+ "\n",
+ "Here we also plot the predicted CTR of this model conditioned on `dollar_rating`. Notice that all the constraints we required are fulfilled in each of the slices."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "rh0H2b6l_rwZ"
+ },
+ "source": [
+ "### Output Calibration"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "colab_type": "text",
+ "id": "KPb2ri4e7HXF"
+ },
+ "source": [
+ "For all the TFL models we have trained so far, the lattice layer (indicated as \"Lattice\" in the model graph) directly outputs the model prediction. Sometimes we are not sure whether the lattice output should be rescaled to emit model outputs:\n",
+ "- the features are $log$ counts while the labels are counts.\n",
+ "- the lattice is configured to have very few vertices but the label distribution is relatively complicated.\n",
+ "\n",
+ "In those cases we can add another calibrator between the lattice output and the model output to increase model flexibility. Here let's add a calibrator layer with 5 keypoints to the model we just built. We also add a regularizer for the output calibrator to keep the function smooth.\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 0,
+ "metadata": {
+ "colab": {},
+ "colab_type": "code",
+ "id": "k5Sg_gUj_0i4"
+ },
+ "outputs": [],
+ "source": [
+ "feature_columns = [\n",
+ " tf.feature_column.numeric_column(\"num_reviews\"),\n",
+ " tf.feature_column.numeric_column(\"avg_rating\"),\n",
+ " tf.feature_column.categorical_column_with_vocabulary_list(\n",
+ " \"dollar_rating\",\n",
+ " vocabulary_list=[\"D\", \"DD\", \"DDD\", \"DDDD\"],\n",
+ " dtype=tf.string,\n",
+ " default_value=0),\n",
+ "]\n",
+ "model_config = tfl.configs.CalibratedLatticeConfig(\n",
+ " output_calibration=True,\n",
+ " output_calibration_num_keypoint=5,\n",
+ " regularizer_configs=[\n",
+ " tfl.configs.RegularizerConfig(name=\"output_calib_wrinkle\", l2=0.1),\n",
+ " ],\n",
+ " feature_configs=[\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"num_reviews\",\n",
+ " lattice_size=2,\n",
+ " monotonicity=\"increasing\",\n",
+ " pwl_calibration_convexity=\"concave\",\n",
+ " pwl_calibration_num_keypoints=20,\n",
+ " regularizer_configs=[\n",
+ " tfl.configs.RegularizerConfig(name=\"calib_wrinkle\", l2=1.0),\n",
+ " ],\n",
+ " reflects_trust_in=[\n",
+ " tfl.configs.TrustConfig(\n",
+ " feature_name=\"avg_rating\", trust_type=\"edgeworth\"),\n",
+ " ],\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"avg_rating\",\n",
+ " lattice_size=2,\n",
+ " monotonicity=\"increasing\",\n",
+ " pwl_calibration_num_keypoints=20,\n",
+ " regularizer_configs=[\n",
+ " tfl.configs.RegularizerConfig(name=\"calib_wrinkle\", l2=1.0),\n",
+ " ],\n",
+ " ),\n",
+ " tfl.configs.FeatureConfig(\n",
+ " name=\"dollar_rating\",\n",
+ " lattice_size=2,\n",
+ " pwl_calibration_num_keypoints=4,\n",
+ " # Here we only specify one monotonicity:\n",
+ " # `D` resturants has smaller value than `DD` restaurants\n",
+ " monotonicity=[(\"D\", \"DD\")],\n",
+ " ),\n",
+ "])\n",
+ "tfl_estimator = tfl.estimators.CannedClassifier(\n",
+ " feature_columns=feature_columns,\n",
+ " model_config=model_config,\n",
+ " feature_analysis_input_fn=feature_analysis_input_fn,\n",
+ " optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),\n",
+ " config=tf.estimator.RunConfig(tf_random_seed=42),\n",
+ ")\n",
+ "tfl_estimator.train(input_fn=train_input_fn)\n",
+ "analyze_three_d_estimator(tfl_estimator, \"TF Lattice\")\n",
+ "_ = save_and_visualize_lattice(tfl_estimator)"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "collapsed_sections": [],
+ "name": "tfl_shape_constraints.ipynb",
+ "private_outputs": true,
+ "provenance": [
+ {
+ "file_id": "1NYk-Kehpe0V3JgdRAYZmR9-kUdKcSxys",
+ "timestamp": 1579632224365
+ }
+ ],
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/examples/BUILD b/examples/BUILD
new file mode 100644
index 0000000..ec47d8d
--- /dev/null
+++ b/examples/BUILD
@@ -0,0 +1,62 @@
+# Copyright 2019 The TensorFlow Lattice Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+licenses(["notice"])
+
+package(
+ default_visibility = [
+ "//tensorflow_lattice:__subpackages__",
+ ],
+)
+
+py_binary(
+ name = "canned_estimators_uci_heart",
+ srcs = ["canned_estimators_uci_heart.py"],
+ python_version = "PY3",
+ deps = [
+ # tensorflow dep,
+ "//tensorflow_lattice",
+ ],
+)
+
+py_binary(
+ name = "keras_sequential_uci_heart",
+ srcs = ["keras_sequential_uci_heart.py"],
+ python_version = "PY3",
+ deps = [
+ # tensorflow dep,
+ "//tensorflow_lattice",
+ ],
+)
+
+py_binary(
+ name = "keras_functional_uci_heart",
+ srcs = ["keras_functional_uci_heart.py"],
+ python_version = "PY3",
+ deps = [
+ # tensorflow dep,
+ "//tensorflow_lattice",
+ ],
+)
+
+py_binary(
+ name = "custom_estimators_uci_heart",
+ srcs = ["custom_estimators_uci_heart.py"],
+ python_version = "PY3",
+ deps = [
+ # tensorflow dep,
+ "//tensorflow_lattice",
+ ],
+)
diff --git a/examples/canned_estimators_uci_heart.py b/examples/canned_estimators_uci_heart.py
new file mode 100644
index 0000000..51597c3
--- /dev/null
+++ b/examples/canned_estimators_uci_heart.py
@@ -0,0 +1,325 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Example usage of TFL canned estimators.
+
+This example trains several TFL canned estimators on the UCI heart dataset.
+
+Example usage:
+canned_estimators_uci_heart --config_updates=feature__age__lattice_size=4
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import re
+from absl import app
+from absl import flags
+import pandas as pd
+import tensorflow as tf
+from tensorflow import feature_column as fc
+from tensorflow_lattice import configs
+from tensorflow_lattice import estimators
+
+FLAGS = flags.FLAGS
+flags.DEFINE_float('learning_rate', 0.1, 'Learning rate.')
+flags.DEFINE_integer('batch_size', 100, 'Batch size.')
+flags.DEFINE_integer('num_epochs', 50, 'Number of training epoch.')
+flags.DEFINE_integer('prefitting_num_epochs', 10, 'Prefitting epochs.')
+flags.DEFINE_list(
+ 'config_updates', '',
+ 'Comma separated list of updates to model configs in name=value format.'
+ 'See tfl.configs.apply_updates().')
+
+
+def main(_):
+ # Parse configs updates from command line flags.
+ config_updates = []
+ for update in FLAGS.config_updates:
+ config_updates.extend(re.findall(r'(\S*)\s*=\s*(\S*)', update))
+
+ # UCI Statlog (Heart) dataset.
+ csv_file = tf.keras.utils.get_file(
+ 'heart.csv', 'http://storage.googleapis.com/applied-dl/heart.csv')
+ df = pd.read_csv(csv_file)
+ target = df.pop('target')
+ train_size = int(len(df) * 0.8)
+ train_x = df[:train_size]
+ train_y = target[:train_size]
+ test_x = df[train_size:]
+ test_y = target[train_size:]
+
+ # feature_analysis_input_fn is used to collect statistics about the input
+ # features, thus requiring only one loop of the dataset.
+ #
+ # feature_analysis_input_fn is required if you have at least one FeatureConfig
+ # with "pwl_calibration_input_keypoints='quantiles'". Note that 'quantiles' is
+ # default keypoints configuration so most likely you'll need it.
+ feature_analysis_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(
+ x=train_x,
+ y=train_y,
+ shuffle=False,
+ batch_size=FLAGS.batch_size,
+ num_epochs=1,
+ num_threads=1)
+
+ # prefitting_input_fn is used to prefit an initial ensemble that is used to
+ # estimate feature interactions. This prefitting step does not need to fully
+ # converge and thus requiring fewer epochs than the main training.
+ #
+ # prefitting_input_fn is only required if your model_config is
+ # CalibratedLatticeEnsembleConfig with "lattices='crystals'"
+ prefitting_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(
+ x=train_x,
+ y=train_y,
+ shuffle=True,
+ batch_size=FLAGS.batch_size,
+ num_epochs=FLAGS.prefitting_num_epochs,
+ num_threads=1)
+
+ train_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(
+ x=train_x,
+ y=train_y,
+ shuffle=True,
+ batch_size=FLAGS.batch_size,
+ num_epochs=FLAGS.num_epochs,
+ num_threads=1)
+
+ test_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(
+ x=test_x,
+ y=test_y,
+ shuffle=False,
+ batch_size=FLAGS.batch_size,
+ num_epochs=FLAGS.num_epochs,
+ num_threads=1)
+
+ # Feature columns.
+ # - age
+ # - sex
+ # - cp chest pain type (4 values)
+ # - trestbps resting blood pressure
+ # - chol serum cholestoral in mg/dl
+ # - fbs fasting blood sugar > 120 mg/dl
+ # - restecg resting electrocardiographic results (values 0,1,2)
+ # - thalach maximum heart rate achieved
+ # - exang exercise induced angina
+ # - oldpeak ST depression induced by exercise relative to rest
+ # - slope the slope of the peak exercise ST segment
+ # - ca number of major vessels (0-3) colored by flourosopy
+ # - thal 3 = normal; 6 = fixed defect; 7 = reversable defect
+ feature_columns = [
+ fc.numeric_column('age', default_value=-1),
+ fc.categorical_column_with_vocabulary_list('sex', [0, 1]),
+ fc.numeric_column('cp'),
+ fc.numeric_column('trestbps', default_value=-1),
+ fc.numeric_column('chol'),
+ fc.categorical_column_with_vocabulary_list('fbs', [0, 1]),
+ fc.categorical_column_with_vocabulary_list('restecg', [0, 1, 2]),
+ fc.numeric_column('thalach'),
+ fc.categorical_column_with_vocabulary_list('exang', [0, 1]),
+ fc.numeric_column('oldpeak'),
+ fc.categorical_column_with_vocabulary_list('slope', [0, 1, 2]),
+ fc.numeric_column('ca'),
+ fc.categorical_column_with_vocabulary_list(
+ 'thal', ['normal', 'fixed', 'reversible']),
+ ]
+
+ # Feature configs are used to specify how each feature is calibrated and used.
+ feature_configs = [
+ configs.FeatureConfig(
+ name='age',
+ lattice_size=3,
+ # By default, input keypoints of pwl are quantiles of the feature.
+ pwl_calibration_num_keypoints=5,
+ monotonicity='increasing',
+ pwl_calibration_clip_max=100,
+ ),
+ configs.FeatureConfig(
+ name='cp',
+ pwl_calibration_num_keypoints=4,
+ # Keypoints can be uniformly spaced.
+ pwl_calibration_input_keypoints='uniform',
+ monotonicity='increasing',
+ ),
+ configs.FeatureConfig(
+ name='chol',
+ # Explicit input keypoint initialization.
+ pwl_calibration_input_keypoints=[126.0, 210.0, 247.0, 286.0, 564.0],
+ monotonicity='increasing',
+ pwl_calibration_clip_min=130,
+ # Calibration can be forced to span the full output range by clamping.
+ pwl_calibration_clamp_min=True,
+ pwl_calibration_clamp_max=True,
+ # Per feature regularization.
+ regularizer_configs=[
+ configs.RegularizerConfig(name='calib_hessian', l2=1e-4),
+ ],
+ ),
+ configs.FeatureConfig(
+ name='fbs',
+ # Monotonicity: output for 1 should be larger than output for 0.
+ monotonicity=[(0, 1)],
+ ),
+ configs.FeatureConfig(
+ name='trestbps',
+ pwl_calibration_num_keypoints=5,
+ monotonicity='decreasing',
+ ),
+ configs.FeatureConfig(
+ name='thalach',
+ pwl_calibration_num_keypoints=5,
+ monotonicity='decreasing',
+ ),
+ configs.FeatureConfig(
+ name='restecg',
+ # Categorical monotonicity can be partial order.
+ monotonicity=[(0, 1), (0, 2)],
+ ),
+ configs.FeatureConfig(
+ name='exang',
+ monotonicity=[(0, 1)],
+ ),
+ configs.FeatureConfig(
+ name='oldpeak',
+ pwl_calibration_num_keypoints=5,
+ monotonicity='increasing',
+ ),
+ configs.FeatureConfig(
+ name='slope',
+ monotonicity=[(0, 1), (1, 2)],
+ ),
+ configs.FeatureConfig(
+ name='ca',
+ pwl_calibration_num_keypoints=4,
+ monotonicity='increasing',
+ ),
+ configs.FeatureConfig(
+ name='thal',
+ monotonicity=[('normal', 'fixed'), ('normal', 'reversible')],
+ ),
+ ]
+
+ # Serving input fn is used to create saved models.
+ serving_input_fn = (
+ tf.estimator.export.build_parsing_serving_input_receiver_fn(
+ feature_spec=fc.make_parse_example_spec(feature_columns)))
+
+ # Model config defines the model strcutre for the estimator.
+ # This is calibrated linear model with outputput calibration: Inputs are
+ # calibrated, linearly combined and the output of the linear layer is
+ # calibrated again using a PWL function.
+ model_config = configs.CalibratedLinearConfig(
+ feature_configs=feature_configs,
+ use_bias=True,
+ output_calibration=True,
+ regularizer_configs=[
+ # Regularizer for the output calibrator.
+ configs.RegularizerConfig(name='output_calib_hessian', l2=1e-4),
+ ])
+ # Update model configuration.
+ # See tfl.configs.apply_updates for details.
+ configs.apply_updates(model_config, config_updates)
+ estimator = estimators.CannedClassifier(
+ feature_columns=feature_columns,
+ model_config=model_config,
+ feature_analysis_input_fn=feature_analysis_input_fn,
+ optimizer=tf.keras.optimizers.Adam(FLAGS.learning_rate))
+ estimator.train(input_fn=train_input_fn)
+ results = estimator.evaluate(input_fn=test_input_fn)
+ print('Calibrated linear results: {}'.format(results))
+ print('Calibrated linear model exported to {}'.format(
+ estimator.export_saved_model(estimator.model_dir, serving_input_fn)))
+
+ # This is calibrated lattice model: Inputs are calibrated, then combined
+ # non-linearly using a lattice layer.
+ model_config = configs.CalibratedLatticeConfig(
+ feature_configs=feature_configs,
+ regularizer_configs=[
+ # Torsion regularizer applied to the lattice to make it more linear.
+ configs.RegularizerConfig(name='torsion', l2=1e-4),
+ # Globally defined calibration regularizer is applied to all features.
+ configs.RegularizerConfig(name='calib_hessian', l2=1e-4),
+ ])
+ estimator = estimators.CannedClassifier(
+ feature_columns=feature_columns,
+ model_config=model_config,
+ feature_analysis_input_fn=feature_analysis_input_fn,
+ optimizer=tf.keras.optimizers.Adam(FLAGS.learning_rate))
+ estimator.train(input_fn=train_input_fn)
+ results = estimator.evaluate(input_fn=test_input_fn)
+ print('Calibrated lattice results: {}'.format(results))
+ print('Calibrated lattice model exported to {}'.format(
+ estimator.export_saved_model(estimator.model_dir, serving_input_fn)))
+
+ # This is random lattice ensemble model with separate calibration:
+ # model output is the average output of separatly calibrated lattices.
+ model_config = configs.CalibratedLatticeEnsembleConfig(
+ feature_configs=feature_configs,
+ num_lattices=6,
+ lattice_rank=5,
+ separate_calibrators=True,
+ regularizer_configs=[
+ # Torsion regularizer applied to the lattice to make it more linear.
+ configs.RegularizerConfig(name='torsion', l2=1e-4),
+ # Globally defined calibration regularizer is applied to all features.
+ configs.RegularizerConfig(name='calib_hessian', l2=1e-4),
+ ])
+ configs.apply_updates(model_config, config_updates)
+ estimator = estimators.CannedClassifier(
+ feature_columns=feature_columns,
+ model_config=model_config,
+ feature_analysis_input_fn=feature_analysis_input_fn,
+ optimizer=tf.keras.optimizers.Adam(FLAGS.learning_rate))
+ estimator.train(input_fn=train_input_fn)
+ results = estimator.evaluate(input_fn=test_input_fn)
+ print('Random ensemble results: {}'.format(results))
+ print('Random ensemble model exported to {}'.format(
+ estimator.export_saved_model(estimator.model_dir, serving_input_fn)))
+
+ # This is Crystals ensemble model with separate calibration: model output is
+ # the average output of separatly calibrated lattices.
+ # Crystals algorithm first trains a prefitting model and uses the interactions
+ # between features to form the final lattice ensemble.
+ model_config = configs.CalibratedLatticeEnsembleConfig(
+ feature_configs=feature_configs,
+ # Using Crystals algorithm.
+ lattices='crystals',
+ num_lattices=6,
+ lattice_rank=5,
+ separate_calibrators=True,
+ regularizer_configs=[
+ # Torsion regularizer applied to the lattice to make it more linear.
+ configs.RegularizerConfig(name='torsion', l2=1e-4),
+ # Globally defined calibration regularizer is applied to all features.
+ configs.RegularizerConfig(name='calib_hessian', l2=1e-4),
+ ])
+ configs.apply_updates(model_config, config_updates)
+ estimator = estimators.CannedClassifier(
+ feature_columns=feature_columns,
+ model_config=model_config,
+ feature_analysis_input_fn=feature_analysis_input_fn,
+ # prefitting_input_fn is required to train the prefitting model.
+ prefitting_input_fn=prefitting_input_fn,
+ optimizer=tf.keras.optimizers.Adam(FLAGS.learning_rate))
+ estimator.train(input_fn=train_input_fn)
+ results = estimator.evaluate(input_fn=test_input_fn)
+ print('Crystals ensemble results: {}'.format(results))
+ print('Crystals ensemble model exported to {}'.format(
+ estimator.export_saved_model(estimator.model_dir, serving_input_fn)))
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/examples/coffee_test.py b/examples/coffee_test.py
deleted file mode 100644
index c465fb3..0000000
--- a/examples/coffee_test.py
+++ /dev/null
@@ -1,84 +0,0 @@
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for lattice estimators."""
-import numpy as np
-import tensorflow as tf
-import tensorflow_lattice as tfl
-
-# Example training and testing data.
-train_features = {
- 'distance': np.array([1.0, 2.0, 3.0, 4.0, 5.0]),
- 'quality': np.array([2.0, 5.0, 1.0, 2.0, 5.0]),
-}
-train_labels = np.array([0.2, 1.0, 0.0, 0.0, 1.0])
-
-# Same quality but different distance.
-test_features = {
- 'distance': np.array([5.0, 10.0]),
- 'quality': np.array([3.0, 3.0]),
-}
-
-# Feature definition.
-feature_columns = [
- tf.feature_column.numeric_column('distance'),
- tf.feature_column.numeric_column('quality'),
-]
-
-# Hyperparameters.
-num_keypoints = 10
-hparams = tfl.CalibratedLatticeHParams(
- feature_names=['distance', 'quality'],
- num_keypoints=num_keypoints,
- learning_rate=0.1,
-)
-
-# Set feature monotonicity.
-hparams.set_feature_param('distance', 'monotonicity', -1)
-hparams.set_feature_param('quality', 'monotonicity', +1)
-
-# Define keypoint init.
-keypoints_init_fns = {
- 'distance': lambda: tfl.uniform_keypoints_for_signal(num_keypoints,
- input_min=0.0,
- input_max=10.0,
- output_min=0.0,
- output_max=1.0),
- 'quality': lambda: tfl.uniform_keypoints_for_signal(num_keypoints,
- input_min=0.0,
- input_max=5.0,
- output_min=0.0,
- output_max=1.0),
-}
-
-lattice_estimator = tfl.calibrated_lattice_regressor(
- feature_columns=feature_columns,
- hparams=hparams,
- keypoints_initializers_fn=keypoints_init_fns)
-
-# Train!
-train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
- x=train_features,
- y=train_labels,
- batch_size=1,
- num_epochs=100,
- shuffle=False)
-
-lattice_estimator.train(input_fn=train_input_fn)
-
-# Test.
-test_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
- x=test_features, y=None, batch_size=1, num_epochs=1, shuffle=False)
-
-print(list(lattice_estimator.predict(input_fn=test_input_fn)))
diff --git a/examples/custom_estimators_uci_heart.py b/examples/custom_estimators_uci_heart.py
new file mode 100644
index 0000000..ac3e5e3
--- /dev/null
+++ b/examples/custom_estimators_uci_heart.py
@@ -0,0 +1,170 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Example usage of TFL layers in custom estimators.
+
+This example trains a TFL custom estimators on the UCI heart dataset.
+
+Example usage:
+custom_estimators_uci_heart --num_epochs=40
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl import app
+from absl import flags
+import numpy as np
+import pandas as pd
+import tensorflow as tf
+from tensorflow import feature_column as fc
+import tensorflow_lattice as tfl
+from tensorflow_estimator.python.estimator.canned import optimizers
+from tensorflow_estimator.python.estimator.head import binary_class_head
+
+FLAGS = flags.FLAGS
+flags.DEFINE_float('learning_rate', 0.01, 'Learning rate.')
+flags.DEFINE_integer('batch_size', 100, 'Batch size.')
+flags.DEFINE_integer('num_epochs', 200, 'Number of training epoch.')
+
+
+def main(_):
+ # UCI Statlog (Heart) dataset.
+ csv_file = tf.keras.utils.get_file(
+ 'heart.csv', 'http://storage.googleapis.com/applied-dl/heart.csv')
+ df = pd.read_csv(csv_file)
+ target = df.pop('target')
+ train_size = int(len(df) * 0.8)
+ train_x = df[:train_size]
+ train_y = target[:train_size]
+ test_x = df[train_size:]
+ test_y = target[train_size:]
+
+ train_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(
+ x=train_x,
+ y=train_y,
+ shuffle=True,
+ batch_size=FLAGS.batch_size,
+ num_epochs=FLAGS.num_epochs,
+ num_threads=1)
+
+ test_input_fn = tf.compat.v1.estimator.inputs.pandas_input_fn(
+ x=test_x,
+ y=test_y,
+ shuffle=False,
+ batch_size=FLAGS.batch_size,
+ num_epochs=FLAGS.num_epochs,
+ num_threads=1)
+
+ # Feature columns.
+ # - age
+ # - sex
+ # - cp chest pain type (4 values)
+ # - trestbps resting blood pressure
+ # - chol serum cholestoral in mg/dl
+ # - fbs fasting blood sugar > 120 mg/dl
+ # - restecg resting electrocardiographic results (values 0,1,2)
+ # - thalach maximum heart rate achieved
+ # - exang exercise induced angina
+ # - oldpeak ST depression induced by exercise relative to rest
+ # - slope the slope of the peak exercise ST segment
+ # - ca number of major vessels (0-3) colored by flourosopy
+ # - thal 3 = normal; 6 = fixed defect; 7 = reversable defect
+ feature_columns = [
+ fc.numeric_column('age', default_value=-1),
+ fc.categorical_column_with_vocabulary_list('sex', [0, 1]),
+ fc.numeric_column('ca'),
+ fc.categorical_column_with_vocabulary_list(
+ 'thal', ['normal', 'fixed', 'reversible']),
+ ]
+
+ def model_fn(features, labels, mode, config):
+ """model_fn for the custom estimator."""
+ del config
+ input_tensors = tfl.estimators.transform_features(features, feature_columns)
+ inputs = {
+ key: tf.keras.layers.Input(shape=(1,), name=key)
+ for key in input_tensors
+ }
+
+ lattice_sizes = [3, 2, 2, 2]
+ lattice_monotonicities = ['increasing', 'none', 'increasing', 'increasing']
+ lattice_input = tf.keras.layers.Concatenate(axis=1)([
+ tfl.layers.PWLCalibration(
+ input_keypoints=np.linspace(10, 100, num=8, dtype=np.float32),
+ # The output range of the calibrator should be the input range of
+ # the following lattice dimension.
+ output_min=0.0,
+ output_max=lattice_sizes[0] - 1.0,
+ monotonicity='increasing',
+ )(inputs['age']),
+ tfl.layers.CategoricalCalibration(
+ # Number of categories including any missing/default category.
+ num_buckets=2,
+ output_min=0.0,
+ output_max=lattice_sizes[1] - 1.0,
+ )(inputs['sex']),
+ tfl.layers.PWLCalibration(
+ input_keypoints=[0.0, 1.0, 2.0, 3.0],
+ output_min=0.0,
+ output_max=lattice_sizes[0] - 1.0,
+ # You can specify TFL regularizers as tuple
+ # ('regularizer name', l1, l2).
+ kernel_regularizer=('hessian', 0.0, 1e-4),
+ monotonicity='increasing',
+ )(inputs['ca']),
+ tfl.layers.CategoricalCalibration(
+ num_buckets=3,
+ output_min=0.0,
+ output_max=lattice_sizes[1] - 1.0,
+ # Categorical monotonicity can be partial order.
+ # (i, j) indicates that we must have output(i) <= output(i).
+ # Make sure to set the lattice monotonicity to 1 for this dimension.
+ monotonicities=[(0, 1), (0, 2)],
+ )(inputs['thal']),
+ ])
+ output = tfl.layers.Lattice(
+ lattice_sizes=lattice_sizes, monotonicities=lattice_monotonicities)(
+ lattice_input)
+
+ training = (mode == tf.estimator.ModeKeys.TRAIN)
+ model = tf.keras.Model(inputs=inputs, outputs=output)
+ logits = model(input_tensors, training=training)
+
+ if training:
+ optimizer = optimizers.get_optimizer_instance_v2('Adam',
+ FLAGS.learning_rate)
+ else:
+ optimizer = None
+
+ head = binary_class_head.BinaryClassHead()
+ return head.create_estimator_spec(
+ features=features,
+ mode=mode,
+ labels=labels,
+ optimizer=optimizer,
+ logits=logits,
+ trainable_variables=model.trainable_variables,
+ update_ops=model.updates)
+
+ estimator = tf.estimator.Estimator(model_fn=model_fn)
+ estimator.train(input_fn=train_input_fn)
+ results = estimator.evaluate(input_fn=test_input_fn)
+ print('Results: {}'.format(results))
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/examples/estimator_test.py b/examples/estimator_test.py
deleted file mode 100644
index f00b063..0000000
--- a/examples/estimator_test.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""A quick test script for TensorFlow Lattice's calibrated RTL estimator."""
-import numpy as np
-
-import tensorflow as tf
-import tensorflow_lattice as tfl
-
-# Feature definition.
-feature_columns = [
- tf.feature_column.numeric_column('x0'),
- tf.feature_column.numeric_column('x1'),
-]
-
-# Hyperparameters.
-num_keypoints = 10
-hparams = tfl.CalibratedRtlHParams(
- num_keypoints=num_keypoints,
- num_lattices=5,
- lattice_rank=2,
- learning_rate=0.1)
-def init_fn():
- return tfl.uniform_keypoints_for_signal(num_keypoints,
- input_min=-1.0,
- input_max=1.0,
- output_min=0.0,
- output_max=1.0)
-
-# Estimator.
-rtl_estimator = tfl.calibrated_rtl_regressor(feature_columns=feature_columns,
- hparams=hparams,
- keypoints_initializers_fn=init_fn)
-
-# Prepare the dataset.
-num_examples = 1000
-x0 = np.random.uniform(-1.0, 1.0, size=num_examples)
-x1 = np.random.uniform(-1.0, 1.0, size=num_examples)
-y = x0 ** 2 + x1 ** 2
-
-# Example input function.
-twod_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
- x={
- 'x0': x0,
- 'x1': x1
- }, y=y, batch_size=10, num_epochs=1, shuffle=False)
-
-# Train!
-rtl_estimator.train(input_fn=twod_input_fn)
-# Evaluate!
-print(rtl_estimator.evaluate(input_fn=twod_input_fn))
diff --git a/examples/etl_1d.py b/examples/etl_1d.py
deleted file mode 100644
index 28ef2a1..0000000
--- a/examples/etl_1d.py
+++ /dev/null
@@ -1,310 +0,0 @@
-# Copyright 2018 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Trains a small (2 inputs, single lattice) on toy data and visualizes it."""
-from __future__ import print_function
-
-import tempfile
-import matplotlib.pyplot as plt
-import numpy as np
-import scipy
-import tensorflow as tf
-import tensorflow_lattice as tfl
-
-np.random.seed(1)
-
-_FEATURE_KEYPOINTS = 'tfl_calibrated_etl/pwl_calibration/X_{}_keypoints_'
-_EMBED_KEYPOINTS = 'tfl_calibrated_etl/non_monotonic_lattices/'
-_EMBED_KEYPOINTS += 'pwl_calibration/signal_{}_keypoints_'
-_LATTICE_PARAMS = 'tfl_calibrated_etl/non_monotonic_lattices/lattice_{}/'
-_LATTICE_PARAMS += 'hypercube_lattice_parameters'
-
-
-def annulus_data(n_points, r_0, r_1):
- """Creates toy dataset in quadrant I with a quarter annulus.
-
- Args:
- n_points: (int) number of points
- r_0: (float) inner bounding radius
- r_1: (float) outer bounding radius
-
- Returns:
- x: (np.Array) covariates
- y: (np.Array) labels
- """
- x = np.random.random(size=(n_points, 2))
- r = (x**2).sum(1)**.5
- y = (r_0 < r) & (r < r_1)
- return x, y.astype(int)
-
-
-def fit_model(x,
- y,
- lattice_size=5,
- non_monotonic_num_lattices=1,
- non_monotonic_lattice_rank=1):
- """Fits a single 1D lattice to the provided data.
-
- Args:
- x: covariates
- y: labels
- lattice_size: (int, optional) Number of knots in each lattice dimension,
- total knots is lattice_size^lattice_rank, for each lattice
- non_monotonic_num_lattices: (int, optional)
- non_monotonic_lattice_rank: (int, optional) number of inputs to each
-
- Returns:
- etl_estimator: fitted TF Estimator
- """
- # Hyperparameters.
- num_keypoints = 100
- hparams = tfl.CalibratedEtlHParams(
- non_monotonic_lattice_rank=non_monotonic_lattice_rank,
- non_monotonic_num_lattices=non_monotonic_num_lattices,
- non_monotonic_lattice_size=lattice_size,
- num_keypoints=num_keypoints,
- learning_rate=0.007,
- linear_embedding_calibration_num_keypoints=100)
-
- # Estimator.
- feature_columns = [
- tf.feature_column.numeric_column('X_0'),
- tf.feature_column.numeric_column('X_1'),
- ]
-
- # Training is sensitive to initialization
- config = tf.estimator.RunConfig(tf_random_seed=1)
- def keypoints_config():
- return tfl.uniform_keypoints_for_signal(
- num_keypoints,
- input_min=0.0,
- input_max=x.max(),
- output_min=0.0,
- output_max=lattice_size - 1
- )
- etl_estimator = tfl.calibrated_etl_classifier(
- feature_columns=feature_columns,
- hparams=hparams,
- keypoints_initializers_fn=keypoints_config,
- config=config
- )
-
- # Input function.
- input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
- x={
- 'X_0': x[:, 0],
- 'X_1': x[:, 1]
- },
- y=y.flatten(),
- batch_size=10000,
- num_epochs=100,
- shuffle=False)
-
- # Train!
- etl_estimator.train(input_fn=input_fn)
-
- # Evaluate
- eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
- x={
- 'X_0': x[:, 0],
- 'X_1': x[:, 1]
- },
- y=y.flatten(),
- batch_size=10000,
- num_epochs=1,
- shuffle=False)
- print(etl_estimator.evaluate(input_fn=eval_input_fn))
-
- return etl_estimator
-
-
-def _get_calibration_params(estimator, dim, weight_key, prefix):
- """Helps extract calibration parameters from TFL graph."""
- input_key = '{}_keypoints_inputs'.format(prefix)
- output_key = '{}_keypoints_outputs'.format(prefix)
- calibrator_key = '{}_calibrators'.format(prefix)
-
- params = {}
- params[input_key], params[output_key], params[calibrator_key] = [], [], []
- for i in xrange(dim):
- params[input_key].append(
- estimator.get_variable_value(weight_key.format(i) + 'inputs'))
- params[output_key].append(
- estimator.get_variable_value(weight_key.format(i) + 'outputs'))
- params[calibrator_key].append(
- scipy.interpolate.interp1d(
- params[input_key][-1],
- params[output_key][-1],
- fill_value='extrapolate'))
- return params
-
-
-def _get_parameters(etl_estimator):
- """Extracts all parameters necessary to evaluate an ETL from estimator."""
- params = {}
- params['embed_weighting'] = etl_estimator.get_variable_value(
- 'tfl_calibrated_etl/linear_embedding/split_non_monotone/monotone_linear'
- '/weight')
- params['embed_bias'] = etl_estimator.get_variable_value(
- 'tfl_calibrated_etl/linear_embedding/split_non_monotone/monotone_linear'
- '/bias')
- params['final_bias'] = etl_estimator.get_variable_value(
- 'tfl_calibrated_etl/ensemble_average/ensemble_bias')
- params['n_embed'] = params['embed_weighting'].shape[0]
- params['n_feature'] = params['embed_weighting'].shape[1]
-
- params.update(
- _get_calibration_params(etl_estimator, params['n_feature'],
- _FEATURE_KEYPOINTS, 'feature'))
-
- params.update(
- _get_calibration_params(
- etl_estimator,
- params['n_embed'],
- _EMBED_KEYPOINTS,
- 'embed',
- ))
-
- n, ws = 0, []
- while _LATTICE_PARAMS.format(n) in etl_estimator.get_variable_names():
- ws.append(etl_estimator.get_variable_value(_LATTICE_PARAMS.format(n)))
- n += 1
- params['lattice_knots'] = np.vstack(ws)
-
- return params
-
-
-def _apply_callibration(x, calibrators):
- x_ = x.copy()
- for n in xrange(x.shape[1]):
- x_[:, n] = calibrators[n](x[:, n])
- return x_
-
-
-def _compress_0_1(x):
- return (x - x.min()) / (x.max() - x.min())
-
-
-def plot_all(etl_estimator, x, y, save_dir):
- """Makes visualizations of ETL Estimator.
-
- Args:
- etl_estimator: (TF ETL Estimator)
- x: (np.Array) inputs
- y: (np.Array) labels, in [0, 1]
- save_dir: (string) directory for saving visualizations
- """
- params = _get_parameters(etl_estimator)
-
- x_cal = _apply_callibration(x, params['feature_calibrators'])
- x_cal_emb = x_cal.dot(params['embed_weighting'].T) + params['embed_bias']
- x_cal_emb_cal = _apply_callibration(x_cal_emb, params['embed_calibrators'])
- x_cal_emb_cal_lat = np.zeros_like(x_cal_emb_cal)
- for i in xrange(params['lattice_knots'].shape[0]):
- interpolator = scipy.interpolate.interp1d(
- range(params['lattice_knots'].shape[1]),
- params['lattice_knots'][i],
- fill_value='extrapolate')
- x_cal_emb_cal_lat[:, i] = interpolator(x_cal_emb_cal[:, i])
-
- predictions = (x_cal_emb_cal_lat.mean(1) + params['final_bias'] >
- .5).astype(int)
-
- plt.figure()
- plt.title('Input Points Colored By Correct Classification')
- plt.scatter(x[:10000, 0], x[:10000, 1], c=y[:10000], alpha=.3)
- plt.savefig(save_dir + '/labeled.png')
-
- for i, (inputs, outputs) in enumerate(
- zip(params['feature_keypoints_inputs'],
- params['feature_keypoints_outputs'])):
- plt.figure()
- plt.title('Calibration Keypoints For Input Column Number {}'.format(i))
- plt.scatter(inputs, outputs)
- plt.savefig(save_dir + '/feature_cal_{}.png'.format(i))
-
- for i, (inputs, outputs) in enumerate(
- zip(params['embed_keypoints_inputs'], params['embed_keypoints_outputs'])):
- plt.figure()
- plt.title('Calibration Keypoints For Emedding Number {}'.format(i))
- plt.scatter(inputs, outputs)
- plt.savefig(save_dir + '/embed_cal_{}.png'.format(i))
-
- for i in xrange(params['lattice_knots'].shape[0]):
- plt.figure()
- plt.title('Lattice knots for lattice number {}'.format(i))
- plt.plot(
- range(params['lattice_knots'].shape[1]), params['lattice_knots'][i])
- plt.savefig(save_dir + '/lattice_{}.png'.format(i))
-
- plt.figure()
- plt.title('Input Points After Calibration, Colored By Correct Classification')
- plt.scatter(x_cal[:10000, 0], x_cal[:10000, 1], c=y[:10000], alpha=.3)
- plt.savefig(save_dir + '/calibrated.png')
-
- plt.figure()
- plt.title('Input Points Colored By Value'
- ' After Calibration and linear transformation')
- plt.scatter(
- x[:10000, 0],
- x[:10000, 1],
- c=_compress_0_1(x_cal_emb[:10000, 0]),
- alpha=.3)
- plt.savefig(save_dir + '/embed_colored.png')
-
- plt.figure()
- plt.title('Input Points Colored By Value After Calibration,'
- '\n Linear Transformation, Second Calibration')
- plt.scatter(
- x[:10000, 0],
- x[:10000, 1],
- c=_compress_0_1(x_cal_emb_cal[:10000, 0]),
- alpha=.3)
- plt.savefig(save_dir + '/embed_calibrated_colored.png')
-
- plt.figure()
- plt.title('Input Points Colored by Value After Calibration,'
- '\nlinear transformation, second calibration, and 1D lattice')
- plt.scatter(
- x[:10000, 0],
- x[:10000, 1],
- c=_compress_0_1(x_cal_emb_cal_lat[:10000, 0]),
- alpha=.3)
- plt.savefig(save_dir + '/lattice_colored.png')
-
- plt.figure()
- plt.title('Predictions')
- plt.scatter(
- x[:10000, 0],
- x[:10000, 1],
- c=_compress_0_1(predictions)[:10000],
- alpha=.3)
- plt.savefig(save_dir + '/predictions.png')
-
-
-def main():
- # Make data
- x, y = annulus_data(300000, .5, .8)
-
- # Train model
- etl_estimator = fit_model(x, y)
-
- # Visualize
- temp_dir = tempfile.mkdtemp()
- print('Saving figures to {}'.format(temp_dir))
- plot_all(etl_estimator, x, y, temp_dir)
-
-
-if __name__ == '__main__':
- main()
diff --git a/examples/image_compression.py b/examples/image_compression.py
deleted file mode 100644
index 02e7885..0000000
--- a/examples/image_compression.py
+++ /dev/null
@@ -1,140 +0,0 @@
-# Copyright 2018 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""A quick example of TensorFlow Lattice's calibrated RTL estimator."""
-from __future__ import print_function
-import sys
-import tempfile
-import matplotlib.pyplot as plt
-import numpy as np
-import tensorflow as tf
-import tensorflow_lattice as tfl
-
-
-def _pixels(im):
- out = np.zeros((im.shape[0] * im.shape[1], 3))
- out[:, 0] = np.repeat(np.arange(im.shape[0]), im.shape[1])
- out[:, 1] = np.tile(np.arange(im.shape[1]), im.shape[0])
- out[:, 2] = im.ravel()
- return out
-
-
-def _pixels_to_image(pixels):
- out = np.zeros((int(pixels[:, 0].max() + 1), int(pixels[:, 1].max() + 1)))
- out[pixels[:, 0].astype(int), pixels[:, 1].astype(int)] = pixels[:, 2]
- return out
-
-
-def run_image(image_path, lattice_size=35):
- """Reads image and fits a 2D lattice to compress it."""
- im = plt.imread(image_path)[:, :, 2]
- im_pixels = _pixels(im)
-
- print('compression ratio is ', lattice_size**2 / float(im.size))
-
- # Hyperparameters.
- num_keypoints = 2
- hparams = tfl.CalibratedRtlHParams(
- num_keypoints=num_keypoints,
- num_lattices=1,
- lattice_rank=2,
- learning_rate=0.003,
- lattice_size=lattice_size)
-
- # Estimator.
- # input: coordinate of the pixel
- # output: value of the pixel
- feature_columns = [
- tf.feature_column.numeric_column('pixel_x'),
- tf.feature_column.numeric_column('pixel_y'),
- ]
-
- def keypoints_initializers():
- return tfl.uniform_keypoints_for_signal(
- num_keypoints,
- input_min=0.0,
- input_max=im_pixels.max(),
- output_min=0.0,
- output_max=lattice_size - 1
- )
- rtl_estimator = tfl.calibrated_rtl_regressor(
- feature_columns=feature_columns,
- hparams=hparams,
- keypoints_initializers_fn=keypoints_initializers
- )
-
- # Example input function.
- input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
- x={
- 'pixel_x': im_pixels[:, 0],
- 'pixel_y': im_pixels[:, 1]
- },
- y=im_pixels[:, 2],
- batch_size=5000,
- num_epochs=15,
- shuffle=True)
-
- # Train!
- rtl_estimator.train(input_fn=input_fn)
-
- # Evaluate!
- eval_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
- x={
- 'pixel_x': im_pixels[:, 0],
- 'pixel_y': im_pixels[:, 1]
- },
- y=im_pixels[:, 2],
- batch_size=5000,
- num_epochs=1,
- shuffle=True)
- print(rtl_estimator.evaluate(input_fn=eval_input_fn))
-
- return rtl_estimator
-
-
-def visualize(estimator, input_img_path, output_dir):
- """Visualizes trained estimator."""
- # This example pulls one channel, also would make sense to convert to gray
- im = plt.imread(input_img_path)[:, :, 2]
- im_pixels = _pixels(im)
-
- input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(
- x={
- 'pixel_x': im_pixels[:, 0],
- 'pixel_y': im_pixels[:, 1]
- },
- batch_size=10000,
- num_epochs=1,
- shuffle=False)
-
- y_test = np.array(
- [q['predictions'] for q in estimator.predict(input_fn=input_fn)])
- img = _pixels_to_image(np.c_[im_pixels[:, :2], y_test])
-
- plt.figure()
- plt.imshow(img, cmap='gray')
- plt.savefig(output_dir + '/image.png')
- return img
-
-
-def main(image_path):
- """Fits image and provides visualization."""
- temp_dir = tempfile.mkdtemp()
- print('Saving output to {}'.format(temp_dir))
- estimator = run_image(image_path)
- visualize(estimator, image_path, temp_dir)
-
-if __name__ == '__main__':
- input_image_path = sys.argv[1]
- main(input_image_path)
diff --git a/examples/keras_functional_uci_heart.py b/examples/keras_functional_uci_heart.py
new file mode 100644
index 0000000..252616e
--- /dev/null
+++ b/examples/keras_functional_uci_heart.py
@@ -0,0 +1,314 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Example usage of TFL within Keras Functional API.
+
+This example builds and trains a calibrated lattice model for the UCI heart
+dataset.
+
+"Calibrated lattice" is a commonly used architecture for datasets where number
+of input features does not exceed ~15.
+
+"Calibrated lattice" assumes every feature being transformed by PWLCalibration
+or CategoricalCalibration layers before nonlineary fusing result of calibration
+within a lattice layer.
+
+The TFL package does not have any layers dedicated to processing of sparse
+features but thanks to plug and play compatibility with any other Keras layers
+we can take advantage of standard Keras embedding to handle sparse features. UCI
+Heart dataset does not have any sparse features so for this example we replaced
+PWLCalibration layer for feature 'age' with Embedding layer in order to
+demonstrate such compatibility as well as advantage of monotonicity
+constraints for semantically meaningful features.
+
+Generally when you manually combine TFL layers you should keep track of:
+1) Ensuring that inputs to TFL layers are within expected range.
+ - Input range for PWLCalibration layer is defined by smallest and largest of
+ provided keypoints.
+ - Input range for Lattice layer is [0.0, lattice_sizes[d] - 1.0] for any
+ dimension d.
+ TFL layers can constraint their output to be within desired range. Feeding
+ output of other layers into TFL layers you might want to ensure that something
+ like sigmoid is used to constraint their output range.
+2) Properly configure monotonicity. If your calibration layer is monotonic then
+ corresponding dimension of lattice layer should also be monotonic.
+
+This example uses functional API for Keras model construction. For an example of
+sequential models with TFL layers see keras_sequential_uci_heart.py.
+
+In order to see how better generalization can be achieved with a properly
+constrained PWLCalibration layer compared to a vanila embedding layer, compare
+training and validation losses of this model with one defined in
+keras_sequential_uci_heart.py
+
+Note that the specifics of layer configurations are for demonstration purposes
+and might not result in optimal performance.
+
+Example usage:
+keras_functional_uci_heart
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl import app
+from absl import flags
+
+import numpy as np
+import pandas as pd
+
+import tensorflow as tf
+from tensorflow import keras
+import tensorflow_lattice as tfl
+
+FLAGS = flags.FLAGS
+flags.DEFINE_integer('num_epochs', 200, 'Number of training epoch.')
+
+
+def main(_):
+ # UCI Statlog (Heart) dataset.
+ csv_file = tf.keras.utils.get_file(
+ 'heart.csv', 'http://storage.googleapis.com/applied-dl/heart.csv')
+ training_data_df = pd.read_csv(csv_file).sample(
+ frac=1.0, random_state=41).reset_index(drop=True)
+
+ # Feature columns.
+ # 0 age
+ # 1 sex
+ # 2 cp chest pain type (4 values)
+ # 3 trestbps resting blood pressure
+ # 4 chol serum cholestoral in mg/dl
+ # 5 fbs fasting blood sugar > 120 mg/dl
+ # 6 restecg resting electrocardiographic results (values 0,1,2)
+ # 7 thalach maximum heart rate achieved
+ # 8 exang exercise induced angina
+ # 9 oldpeak ST depression induced by exercise relative to rest
+ # 10 slope the slope of the peak exercise ST segment
+ # 11 ca number of major vessels (0-3) colored by flourosopy
+ # 12 thal 3 = normal; 6 = fixed defect; 7 = reversable defect
+
+ # Example slice of training data:
+ # age sex cp trestbps chol fbs restecg thalach exang oldpeak
+ # 0 63 1 1 145 233 1 2 150 0 2.3
+ # 1 67 1 4 160 286 0 2 108 1 1.5
+ # 2 67 1 4 120 229 0 2 129 1 2.6
+ # 3 37 1 3 130 250 0 0 187 0 3.5
+ # 4 41 0 2 130 204 0 2 172 0 1.4
+ # 5 56 1 2 120 236 0 0 178 0 0.8
+ # 6 62 0 4 140 268 0 2 160 0 3.6
+ # 7 57 0 4 120 354 0 0 163 1 0.6
+ # 8 63 1 4 130 254 0 2 147 0 1.4
+ # 9 53 1 4 140 203 1 2 155 1 3.1
+
+ model_inputs = []
+ lattice_inputs = []
+ # We are going to have 2-d embedding as one of lattice inputs.
+ lattice_sizes_for_embedding = [2, 3]
+ lattice_sizes = lattice_sizes_for_embedding + [2, 2, 3, 3, 2, 2]
+
+ # ############### age ###############
+
+ age_input = keras.layers.Input(shape=[1])
+ model_inputs.append(age_input)
+ age_embedding = keras.layers.Embedding(
+ input_dim=10,
+ output_dim=len(lattice_sizes_for_embedding),
+ embeddings_initializer=keras.initializers.RandomNormal(seed=1)
+ )(age_input)
+ # Flatten to get rid of redundant tensor dimension created by embedding layer.
+ age_embedding = keras.layers.Flatten()(age_embedding)
+
+ # Lattice expects input data for lattice dimension d to be within
+ # [0, lattice_sizes[d]-1.0]. Apply sigmoid and multiply it by input range to
+ # ensure that lattice inputs are within expected range.
+ embedding_lattice_input_range = tf.constant(
+ [size - 1.0 for size in lattice_sizes_for_embedding],
+ # Insert dimension of size 1 in front to ensure that batch dimension
+ # will not collapse as result of multiplication.
+ shape=(1, 2))
+ age_ranged = keras.layers.multiply(
+ [keras.activations.sigmoid(age_embedding),
+ embedding_lattice_input_range])
+ lattice_inputs.append(age_ranged)
+
+ # ############### sex ###############
+
+ # For boolean features simply specify CategoricalCalibration layer with 2
+ # buckets.
+ sex_input = keras.layers.Input(shape=[1])
+ model_inputs.append(sex_input)
+ sex_calibrator = tfl.layers.CategoricalCalibration(
+ num_buckets=2,
+ output_min=0.0,
+ output_max=lattice_sizes[2] - 1.0,
+ # Initializes all outputs to (output_min + output_max) / 2.0.
+ kernel_initializer='constant',
+ )(sex_input)
+ lattice_inputs.append(sex_calibrator)
+
+ # ############### cp ###############
+
+ cp_input = keras.layers.Input(shape=[1])
+ model_inputs.append(cp_input)
+ cp_calibrator = tfl.layers.PWLCalibration(
+ # Here instead of specifying dtype of layer we convert keypoints into
+ # np.float32.
+ input_keypoints=np.linspace(1, 4, num=4, dtype=np.float32),
+ output_min=0.0,
+ output_max=lattice_sizes[3] - 1.0,
+ monotonicity='increasing',
+ # You can specify TFL regularizers as tuple ('regularizer name', l1, l2).
+ kernel_regularizer=('hessian', 0.0, 1e-4)
+ )(cp_input)
+ lattice_inputs.append(cp_calibrator)
+
+ # ############### trestbps ###############
+
+ trestbps_input = keras.layers.Input(shape=[1])
+ model_inputs.append(trestbps_input)
+ trestbps_calibrator = tfl.layers.PWLCalibration(
+ # Alternatively to uniform keypoints you might want to use quantiles as
+ # keypoints.
+ input_keypoints=np.quantile(
+ training_data_df['trestbps'], np.linspace(0.0, 1.0, num=5)),
+ dtype=tf.float32,
+ # Together with quantile keypoints you might want to initialize piecewise
+ # linear function to have 'equal_slopes' in order for output of layer
+ # after initialization to preserve original distribution.
+ kernel_initializer='equal_slopes',
+ output_min=0.0,
+ output_max=lattice_sizes[4] - 1.0,
+ # You might consider clamping extreme inputs of the calibrator to output
+ # bounds.
+ clamp_min=True,
+ clamp_max=True,
+ monotonicity='increasing',
+ )(trestbps_input)
+ lattice_inputs.append(trestbps_calibrator)
+
+ # ############### chol ###############
+
+ chol_input = keras.layers.Input(shape=[1])
+ model_inputs.append(chol_input)
+ chol_calibrator = tfl.layers.PWLCalibration(
+ # Explicit input keypoint initialization.
+ input_keypoints=[126.0, 210.0, 247.0, 286.0, 564.0],
+ output_min=0.0,
+ output_max=lattice_sizes[5] - 1.0,
+ # Monotonicity of calibrator can be decreasing. Note that corresponding
+ # lattice dimension must have INCREASING monotonicity regardless of
+ # monotonicity direction of calibrator.
+ # Its not some weird configuration hack. Its just how math works :)
+ monotonicity='decreasing',
+ # Convexity together with decreasing monotonicity result in diminishing
+ # return constraint.
+ convexity='convex',
+ # You can specify list of regularizers. You are not limited to TFL
+ # regularizrs. Feel free to use any :)
+ kernel_regularizer=[('laplacian', 0.0, 1e-4),
+ keras.regularizers.l1_l2(l1=0.001)]
+ )(chol_input)
+ lattice_inputs.append(chol_calibrator)
+
+ # ############### fbs ###############
+
+ fbs_input = keras.layers.Input(shape=[1])
+ model_inputs.append(fbs_input)
+ fbs_calibrator = tfl.layers.CategoricalCalibration(
+ num_buckets=2,
+ output_min=0.0,
+ output_max=lattice_sizes[6] - 1.0,
+ # For categorical calibration layer monotonicity is specified for pairs
+ # of indices of categories. Output for first category in pair will be
+ # smaller than output for second category.
+ #
+ # Don't forget to set monotonicity of corresponding dimension of Lattice
+ # layer to 'increasing'.
+ monotonicities=[(0, 1)],
+ # This initializer is identical to default one ('uniform'), but has fixed
+ # seed in order to simplify experimentation.
+ kernel_initializer=keras.initializers.RandomUniform(
+ minval=0.0, maxval=lattice_sizes[5] - 1.0, seed=1),
+ )(fbs_input)
+ lattice_inputs.append(fbs_calibrator)
+
+ # ############### restecg ###############
+
+ restecg_input = keras.layers.Input(shape=[1])
+ model_inputs.append(restecg_input)
+ restecg_calibrator = tfl.layers.CategoricalCalibration(
+ num_buckets=3,
+ output_min=0.0,
+ output_max=lattice_sizes[7] - 1.0,
+ # Categorical monotonicity can be partial order.
+ monotonicities=[(0, 1), (0, 2)],
+ # Categorical calibration layer supports standard Keras regularizers.
+ kernel_regularizer=keras.regularizers.l1_l2(l1=0.001),
+ kernel_initializer='constant',
+ )(restecg_input)
+ lattice_inputs.append(restecg_calibrator)
+
+ # Lattice inputs must be either list of d tensors of rank (batch_size, 1) or
+ # single tensor of rank (batch_size, d) where d is dimensionality of lattice.
+ # Since our embedding layer has size 2 in second dimension - concatenate all
+ # of inputs to create single tensor.
+ lattice_inputs_tensor = keras.layers.concatenate(lattice_inputs, axis=1)
+
+ # Create Lattice layer to nonlineary fuse output of calibrators. Don't forget
+ # to specify 'increasing' monotonicity for any dimension for which
+ # monotonicity is configured regardless of monotonicity direction of those.
+ # This includes partial monotonicity of CategoricalCalibration layer.
+ # Note that making embedding inputs monotonic does not make sense.
+ lattice = tfl.layers.Lattice(
+ lattice_sizes=lattice_sizes,
+ monotonicities=['none', 'none', 'none', 'increasing', 'increasing',
+ 'increasing', 'increasing', 'increasing'],
+ output_min=0.0,
+ output_max=1.0,
+ )(lattice_inputs_tensor)
+
+ model = keras.models.Model(
+ inputs=model_inputs,
+ outputs=lattice)
+ model.compile(loss=keras.losses.mean_squared_error,
+ optimizer=keras.optimizers.Adagrad(learning_rate=1.0))
+
+ feature_names = ['age', 'sex', 'cp', 'trestbps', 'chol', 'fbs', 'restecg']
+ features = np.split(training_data_df[feature_names].values.astype(np.float32),
+ indices_or_sections=len(feature_names),
+ axis=1)
+ target = training_data_df[['target']].values.astype(np.float32)
+
+ # Bucketize input for embedding.
+ embedding_bins = np.quantile(
+ features[0],
+ # 10 keypoints will produce 9 dims numbered 1..9 to match embedding input
+ # size of 10.
+ np.linspace(0.0, 1.0, num=10, dtype=np.float32))
+ # Ensure that highest age will get into last bin rather than its own one.
+ embedding_bins[-1] += 1.0
+ features[0] = np.digitize(features[0], bins=embedding_bins)
+
+ model.fit(features,
+ target,
+ batch_size=32,
+ epochs=FLAGS.num_epochs,
+ validation_split=0.2,
+ shuffle=False)
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/examples/keras_sequential_uci_heart.py b/examples/keras_sequential_uci_heart.py
new file mode 100644
index 0000000..3c721ec
--- /dev/null
+++ b/examples/keras_sequential_uci_heart.py
@@ -0,0 +1,275 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Example usage of TFL within Keras models.
+
+This example builds and trains a calibrated lattice model for the UCI heart
+dataset.
+
+"Calibrated lattice" is a commonly used architecture for datasets where number
+of input features does not exceed ~15.
+
+"Calibrated lattice" assumes every feature being transformed by PWLCalibration
+or CategoricalCalibration layers before nonlineary fusing result of calibration
+within a lattice layer.
+
+Generally when you manually combine TFL layers you should keep track of:
+1) Ensuring that inputs to TFL layers are within expected range.
+ - Input range for PWLCalibration layer is defined by smallest and largest of
+ provided keypoints.
+ - Input range for Lattice layer is [0.0, lattice_sizes[d] - 1.0] for any
+ dimension d.
+ TFL layers can constraint their output to be within desired range. Feeding
+ output of other layers into TFL layers you might want to ensure that something
+ like sigmoid is used to constraint their output range.
+2) Properly configure monotonicity. If your calibration layer is monotonic then
+ corresponding dimension of lattice layer should also be monotonic.
+
+This example creates a Sequential Keras model and only uses TFL layers. For an
+example of functional model construction that also use embedding layers see
+keras_functional_uci_heart.py.
+
+In order to see how better generalization can be achieved with a properly
+constrained PWLCalibration layer compared to a vanila embedding layer, compare
+training and validation losses of this model with one defined in
+keras_functional_uci_heart.py
+
+
+Note that the specifics of layer configurations are for demonstration purposes
+and might not result in optimal performance.
+
+Example usage:
+keras_sequential_uci_heart
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl import app
+from absl import flags
+
+import numpy as np
+import pandas as pd
+
+import tensorflow as tf
+from tensorflow import keras
+import tensorflow_lattice as tfl
+
+FLAGS = flags.FLAGS
+flags.DEFINE_integer('num_epochs', 200, 'Number of training epoch.')
+
+
+def main(_):
+ # UCI Statlog (Heart) dataset.
+ csv_file = tf.keras.utils.get_file(
+ 'heart.csv', 'http://storage.googleapis.com/applied-dl/heart.csv')
+ training_data_df = pd.read_csv(csv_file).sample(
+ frac=1.0, random_state=41).reset_index(drop=True)
+
+ # Feature columns.
+ # 0 age
+ # 1 sex
+ # 2 cp chest pain type (4 values)
+ # 3 trestbps resting blood pressure
+ # 4 chol serum cholestoral in mg/dl
+ # 5 fbs fasting blood sugar > 120 mg/dl
+ # 6 restecg resting electrocardiographic results (values 0,1,2)
+ # 7 thalach maximum heart rate achieved
+ # 8 exang exercise induced angina
+ # 9 oldpeak ST depression induced by exercise relative to rest
+ # 10 slope the slope of the peak exercise ST segment
+ # 11 ca number of major vessels (0-3) colored by flourosopy
+ # 12 thal 3 = normal; 6 = fixed defect; 7 = reversable defect
+
+ # Example slice of training data:
+ # age sex cp trestbps chol fbs restecg thalach exang oldpeak
+ # 0 63 1 1 145 233 1 2 150 0 2.3
+ # 1 67 1 4 160 286 0 2 108 1 1.5
+ # 2 67 1 4 120 229 0 2 129 1 2.6
+ # 3 37 1 3 130 250 0 0 187 0 3.5
+ # 4 41 0 2 130 204 0 2 172 0 1.4
+ # 5 56 1 2 120 236 0 0 178 0 0.8
+ # 6 62 0 4 140 268 0 2 160 0 3.6
+ # 7 57 0 4 120 354 0 0 163 1 0.6
+ # 8 63 1 4 130 254 0 2 147 0 1.4
+ # 9 53 1 4 140 203 1 2 155 1 3.1
+
+ # Lattice sizes per dimension for Lattice layer.
+ # Lattice layer expects input[i] to be within [0, lattice_sizes[i] - 1.0], so
+ # we need to define lattice sizes ahead of calibration layers so we can
+ # properly specify output range of calibration layers.
+ lattice_sizes = [3, 2, 2, 2, 2, 2, 2]
+
+ # Use ParallelCombination helper layer to group togehter calibration layers
+ # which have to be executed in paralel in order to be able to use Sequential
+ # model. Alternatively you can use functional API.
+ combined_calibrators = tfl.layers.ParallelCombination()
+
+ # Configure calibration layers for every feature:
+
+ # ############### age ###############
+
+ calibrator = tfl.layers.PWLCalibration(
+ # Every PWLCalibration layer must have keypoints of piecewise linear
+ # function specified. Easiest way to specify them is to uniformly cover
+ # entire input range by using numpy.linspace().
+ input_keypoints=np.linspace(training_data_df['age'].min(),
+ training_data_df['age'].max(),
+ num=5),
+ # You need to ensure that input keypoints have same dtype as layer input.
+ # You can do it by setting dtype here or by providing keypoints in such
+ # format which will be converted to deisred tf.dtype by default.
+ dtype=tf.float32,
+ # Output range must correspond to expected lattice input range.
+ output_min=0.0,
+ output_max=lattice_sizes[0] - 1.0,
+ monotonicity='increasing')
+ combined_calibrators.append(calibrator)
+
+ # ############### sex ###############
+
+ # For boolean features simply specify CategoricalCalibration layer with 2
+ # buckets.
+ calibrator = tfl.layers.CategoricalCalibration(
+ num_buckets=2,
+ output_min=0.0,
+ output_max=lattice_sizes[1] - 1.0,
+ # Initializes all outputs to (output_min + output_max) / 2.0.
+ kernel_initializer='constant')
+ combined_calibrators.append(calibrator)
+
+ # ############### cp ###############
+
+ calibrator = tfl.layers.PWLCalibration(
+ # Here instead of specifying dtype of layer we convert keypoints into
+ # np.float32.
+ input_keypoints=np.linspace(1, 4, num=4, dtype=np.float32),
+ output_min=0.0,
+ output_max=lattice_sizes[2] - 1.0,
+ monotonicity='increasing',
+ # You can specify TFL regularizers as tuple ('regularizer name', l1, l2).
+ kernel_regularizer=('hessian', 0.0, 1e-4))
+ combined_calibrators.append(calibrator)
+
+ # ############### trestbps ###############
+
+ calibrator = tfl.layers.PWLCalibration(
+ # Alternatively to uniform keypoints you might want to use quantiles as
+ # keypoints.
+ input_keypoints=np.quantile(
+ training_data_df['trestbps'], np.linspace(0.0, 1.0, num=5)),
+ dtype=tf.float32,
+ # Together with quantile keypoints you might want to initialize piecewise
+ # linear function to have 'equal_slopes' in order for output of layer
+ # after initialization to preserve original distribution.
+ kernel_initializer='equal_slopes',
+ output_min=0.0,
+ output_max=lattice_sizes[3] - 1.0,
+ # You might consider clamping extreme inputs of the calibrator to output
+ # bounds.
+ clamp_min=True,
+ clamp_max=True,
+ monotonicity='increasing')
+ combined_calibrators.append(calibrator)
+
+ # ############### chol ###############
+
+ calibrator = tfl.layers.PWLCalibration(
+ # Explicit input keypoint initialization.
+ input_keypoints=[126.0, 210.0, 247.0, 286.0, 564.0],
+ dtype=tf.float32,
+ output_min=0.0,
+ output_max=lattice_sizes[4] - 1.0,
+ # Monotonicity of calibrator can be 'decreasing'. Note that corresponding
+ # lattice dimension must have 'increasing' monotonicity regardless of
+ # monotonicity direction of calibrator.
+ # Its not some weird configuration hack. Its just how math works :)
+ monotonicity='decreasing',
+ # Convexity together with decreasing monotonicity result in diminishing
+ # return constraint.
+ convexity='convex',
+ # You can specify list of regularizers. You are not limited to TFL
+ # regularizrs. Feel free to use any :)
+ kernel_regularizer=[('laplacian', 0.0, 1e-4),
+ keras.regularizers.l1_l2(l1=0.001)])
+ combined_calibrators.append(calibrator)
+
+ # ############### fbs ###############
+
+ calibrator = tfl.layers.CategoricalCalibration(
+ num_buckets=2,
+ output_min=0.0,
+ output_max=lattice_sizes[5] - 1.0,
+ # For categorical calibration layer monotonicity is specified for pairs
+ # of indices of categories. Output for first category in pair will be
+ # smaller than output for second category.
+ #
+ # Don't forget to set monotonicity of corresponding dimension of Lattice
+ # layer to 'increasing'.
+ monotonicities=[(0, 1)],
+ # This initializer is identical to default one('uniform'), but has fixed
+ # seed in order to simplify experimentation.
+ kernel_initializer=keras.initializers.RandomUniform(
+ minval=0.0, maxval=lattice_sizes[5] - 1.0, seed=1))
+ combined_calibrators.append(calibrator)
+
+ # ############### restecg ###############
+
+ calibrator = tfl.layers.CategoricalCalibration(
+ num_buckets=3,
+ output_min=0.0,
+ output_max=lattice_sizes[6] - 1.0,
+ # Categorical monotonicity can be partial order.
+ monotonicities=[(0, 1), (0, 2)],
+ # Categorical calibration layer supports standard Keras regularizers.
+ kernel_regularizer=keras.regularizers.l1_l2(l1=0.001),
+ kernel_initializer='constant')
+ combined_calibrators.append(calibrator)
+
+ # Create Lattice layer to nonlineary fuse output of calibrators. Don't forget
+ # to specify monotonicity 'increasing' for any dimension which calibrator is
+ # monotonic regardless of monotonicity direction of calibrator. This includes
+ # partial monotonicity of CategoricalCalibration layer.
+ lattice = tfl.layers.Lattice(
+ lattice_sizes=lattice_sizes,
+ monotonicities=['increasing', 'none', 'increasing', 'increasing',
+ 'increasing', 'increasing', 'increasing'],
+ output_min=0.0,
+ output_max=1.0)
+
+ model = keras.models.Sequential()
+ # We have just 2 layer as far as Sequential model is concerned.
+ # PWLConcatenate layer takes care of grouping calibrators.
+ model.add(combined_calibrators)
+ model.add(lattice)
+ model.compile(loss=keras.losses.mean_squared_error,
+ optimizer=keras.optimizers.Adagrad(learning_rate=1.0))
+
+ features = training_data_df[
+ ['age', 'sex', 'cp',
+ 'trestbps', 'chol', 'fbs', 'restecg']].values.astype(np.float32)
+ target = training_data_df[['target']].values.astype(np.float32)
+
+ model.fit(features,
+ target,
+ batch_size=32,
+ epochs=FLAGS.num_epochs,
+ validation_split=0.2,
+ shuffle=False)
+
+
+if __name__ == '__main__':
+ app.run(main)
diff --git a/examples/lattice_test.py b/examples/lattice_test.py
deleted file mode 100644
index a441086..0000000
--- a/examples/lattice_test.py
+++ /dev/null
@@ -1,28 +0,0 @@
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""A quick test script for TensorFlow Lattice's lattice layer."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import tensorflow as tf
-import tensorflow_lattice as tfl
-
-x = tf.compat.v1.placeholder(tf.float32, shape=(None, 2))
-(y, _, _, _) = tfl.lattice_layer(x, lattice_sizes=(2, 2))
-
-with tf.Session() as sess:
- sess.run(tf.global_variables_initializer())
- print(sess.run(y, feed_dict={x: [[0.0, 0.0]]}))
diff --git a/examples/uci_census.py b/examples/uci_census.py
deleted file mode 100644
index 55bbc5a..0000000
--- a/examples/uci_census.py
+++ /dev/null
@@ -1,557 +0,0 @@
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Train and evaluate models on UCI Census data.
-
-This is an example TensorFlow Lattice model training and evaluating program,
-using TensorFlow's `tf.estimators` library, a high level abstraction layer
-for machine learning models.
-
-TensorFlow Lattice also offers "layer" level components, so one can customize
-their own models, but these are not included in this example.
-
-Example run for calibrated linear model:
-
-* Uses bash variables `type` and `attempt` for convenience. You can bump
- `attempt` when trying different hyper-parameters.
-* The flag `--create_quantiles` need to be set just the very first time you
- run, since the data quantiles information used for calibration is the same
- for all models.
-* Use `--hparams` to set changes to default parameters.
-* It will print out evaluation on the training data and evaluation data
- every 1/10th of the training epochs.
-
-```bash
-$ type=calibrated_linear ; attempt=1 ;
- python uci_census.py --run=train --model_type=${type}
- --output_dir=${HOME}/experiments/uci_census/${type}_${attempt}
- --quantiles_dir=${HOME}/experiments/uci_census
- --train_epochs=600 --batch_size=1000
- --hparams=learning_rate=1e-3
- --create_quantiles
-```
-
-Example run for calibrated RTL model (assumes you already created the
-quantiles):
-
-* Notice calibrated RTL models train slower than calibrated linear model, but
-should yield slightly better results.
-
-```bash
-$ type=calibrated_rtl ; attempt=1 ;
- python uci_census.py --run=train --model_type=${type}
- --output_dir=${HOME}/experiments/uci_census/${type}_${attempt}
- --quantiles_dir=${HOME}/experiments/uci_census
- --train_epochs=600 --batch_size=1000
- --hparams=learning_rate=1e-2
-```
-
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import tempfile
-
-import pandas as pd
-import six
-import tensorflow as tf
-import tensorflow_lattice as tfl
-
-flags = tf.flags
-FLAGS = flags.FLAGS
-
-# Run mode of the program.
-flags.DEFINE_string(
- "run", "train", "One of 'train', 'evaluate' or 'save', train will "
- "train on training data and also optionally evaluate; evaluate will "
- "evaluate train and test data; save saves the trained model so far "
- "so it can be used by TensorFlow Serving.")
-
-# Dataset.
-flags.DEFINE_string("test", "/tmp/uci_census/adult.test", "Path to test file.")
-flags.DEFINE_string("train", "/tmp/uci_census/adult.data",
- "Path to train file.")
-
-# Model flags.
-flags.DEFINE_string(
- "output_dir", None,
- "Directory where to store the model. If not set a temporary directory "
- "will be automatically created.")
-flags.DEFINE_string(
- "model_type", "calibrated_linear",
- "Types defined in this example: calibrated_linear, calibrated_lattice, "
- " calibrated_rtl, calibrated_etl, calibrated_dnn")
-flags.DEFINE_integer("batch_size", 1000,
- "Number of examples to include in one batch. Increase "
- "this number to improve parallelism, at cost of memory.")
-flags.DEFINE_string("hparams", None,
- "Model hyperparameters, see hyper-parameters in Tensorflow "
- "Lattice documentation. Example: --hparams=learning_rate="
- "0.1,lattice_size=2,num_keypoints=100")
-
-# Calibration quantiles flags.
-flags.DEFINE_bool("create_quantiles", False,
- "Run once to create histogram of features for calibration. "
- "It will use the --train dataset for that.")
-flags.DEFINE_string(
- "quantiles_dir", None,
- "Directory where to store quantile information, defaults to the model "
- "directory (set by --output-dir) but since quantiles can be reused by "
- "models with different parameters, you may want to have a separate "
- "directory.")
-
-# Training flags.
-flags.DEFINE_integer("train_epochs", 10,
- "How many epochs over data during training.")
-flags.DEFINE_bool(
- "train_evaluate_on_train", True,
- "If set, every 1/10th of the train_epochs runs an evaluation on the "
- "full train data.")
-flags.DEFINE_bool(
- "train_evaluate_on_test", True,
- "If set, every 1/10th of the train_epochs runs an evaluation on the "
- "full test data.")
-
-# Columns in dataset files.
-CSV_COLUMNS = [
- "age", "workclass", "fnlwgt", "education", "education_num",
- "marital_status", "occupation", "relationship", "race", "gender",
- "capital_gain", "capital_loss", "hours_per_week", "native_country",
- "income_bracket"
-]
-
-
-def get_test_input_fn(batch_size, num_epochs, shuffle):
- return get_input_fn(FLAGS.test, batch_size, num_epochs, shuffle)
-
-
-def get_train_input_fn(batch_size, num_epochs, shuffle):
- return get_input_fn(FLAGS.train, batch_size, num_epochs, shuffle)
-
-
-# Copy of data read from train/test files: keep copy to avoid re-reading
-# it at every training/evaluation loop.
-_df_data = {}
-_df_data_labels = {}
-
-
-def get_input_fn(file_path, batch_size, num_epochs, shuffle):
- """Returns an input_fn closure for given parameters."""
- if file_path not in _df_data:
- _df_data[file_path] = pd.read_csv(
- tf.gfile.Open(file_path),
- names=CSV_COLUMNS,
- skipinitialspace=True,
- engine="python",
- skiprows=1)
- _df_data[file_path] = _df_data[file_path].dropna(how="any", axis=0)
- _df_data_labels[file_path] = _df_data[file_path]["income_bracket"].apply(
- lambda x: ">50K" in x).astype(int)
- return tf.compat.v1.estimator.inputs.pandas_input_fn(
- x=_df_data[file_path],
- y=_df_data_labels[file_path],
- batch_size=batch_size,
- shuffle=shuffle,
- num_epochs=num_epochs,
- num_threads=1)
-
-
-def create_feature_columns():
- """Creates feature columns for UCI Census, some are sparse."""
- # Categorical features.
- gender = tf.feature_column.categorical_column_with_vocabulary_list(
- "gender", ["Female", "Male"])
- education = tf.feature_column.categorical_column_with_vocabulary_list(
- "education", [
- "Bachelors", "HS-grad", "11th", "Masters", "9th", "Some-college",
- "Assoc-acdm", "Assoc-voc", "7th-8th", "Doctorate", "Prof-school",
- "5th-6th", "10th", "1st-4th", "Preschool", "12th"
- ])
- marital_status = tf.feature_column.categorical_column_with_vocabulary_list(
- "marital_status", [
- "Married-civ-spouse", "Divorced", "Married-spouse-absent",
- "Never-married", "Separated", "Married-AF-spouse", "Widowed"
- ])
- relationship = tf.feature_column.categorical_column_with_vocabulary_list(
- "relationship", [
- "Husband", "Not-in-family", "Wife", "Own-child", "Unmarried",
- "Other-relative"
- ])
- workclass = tf.feature_column.categorical_column_with_vocabulary_list(
- "workclass", [
- "Self-emp-not-inc", "Private", "State-gov", "Federal-gov",
- "Local-gov", "?", "Self-emp-inc", "Without-pay", "Never-worked"
- ])
- occupation = tf.feature_column.categorical_column_with_vocabulary_list(
- "occupation", [
- "Prof-specialty", "Craft-repair", "Exec-managerial", "Adm-clerical",
- "Sales", "Other-service", "Machine-op-inspct", "?",
- "Transport-moving", "Handlers-cleaners", "Farming-fishing",
- "Tech-support", "Protective-serv", "Priv-house-serv", "Armed-Forces"
- ])
- race = tf.feature_column.categorical_column_with_vocabulary_list(
- "race", [
- "White",
- "Black",
- "Asian-Pac-Islander",
- "Amer-Indian-Eskimo",
- "Other",
- ])
- native_country = tf.feature_column.categorical_column_with_vocabulary_list(
- "native_country", [
- "United-States",
- "Mexico",
- "?",
- "Philippines",
- "Germany",
- "Canada",
- "Puerto-Rico",
- "El-Salvador",
- "India",
- "Cuba",
- "England",
- "Jamaica",
- "South",
- "China",
- "Italy",
- "Dominican-Republic",
- "Vietnam",
- "Guatemala",
- "Japan",
- "Poland",
- "Columbia",
- "Taiwan",
- "Haiti",
- "Iran",
- "Portugal",
- "Nicaragua",
- "Peru",
- "Greece",
- "France",
- "Ecuador",
- "Ireland",
- "Hong",
- "Trinadad&Tobago",
- "Cambodia",
- "Thailand",
- "Laos",
- "Yugoslavia",
- "Outlying-US(Guam-USVI-etc)",
- "Hungary",
- "Honduras",
- "Scotland",
- "Holand-Netherlands",
- ])
-
- # Numerical (continuous) base columns.
- age = tf.feature_column.numeric_column("age")
- education_num = tf.feature_column.numeric_column("education_num")
- capital_gain = tf.feature_column.numeric_column("capital_gain")
- capital_loss = tf.feature_column.numeric_column("capital_loss")
- hours_per_week = tf.feature_column.numeric_column("hours_per_week")
-
- # fnlwgt: this should be the weight, how representative this example is of
- # the population, we don't use it here.
- # fnlwgt = tf.feature_column.numeric_column("fnlwgt")
-
- # income-bracket is the label, so, not returned here.
- return [
- age,
- workclass,
- education,
- education_num,
- marital_status,
- occupation,
- relationship,
- race,
- gender,
- capital_gain,
- capital_loss,
- hours_per_week,
- native_country,
- ]
-
-
-def create_quantiles(quantiles_dir):
- """Creates quantiles directory if it doesn't yet exist."""
- batch_size = 10000
- input_fn = get_train_input_fn(
- batch_size=batch_size, num_epochs=1, shuffle=False)
- # Reads until input is exhausted, 10000 at a time.
- tfl.save_quantiles_for_keypoints(
- input_fn=input_fn,
- save_dir=quantiles_dir,
- feature_columns=create_feature_columns(),
- num_steps=None)
-
-
-def _pprint_hparams(hparams):
- """Pretty-print hparams."""
- print("* hparams=[")
- for (key, value) in sorted(six.iteritems(hparams.values())):
- print("\t{}={}".format(key, value))
- print("]")
-
-
-def create_calibrated_linear(feature_columns, config, quantiles_dir):
- feature_names = [fc.name for fc in feature_columns]
- hparams = tfl.CalibratedLinearHParams(
- feature_names=feature_names, num_keypoints=200, learning_rate=1e-4)
- hparams.parse(FLAGS.hparams)
- hparams.set_feature_param("capital_gain", "calibration_l2_laplacian_reg",
- 4.0e-3)
- _pprint_hparams(hparams)
- return tfl.calibrated_linear_classifier(
- feature_columns=feature_columns,
- model_dir=config.model_dir,
- config=config,
- hparams=hparams,
- quantiles_dir=quantiles_dir)
-
-
-def create_calibrated_lattice(feature_columns, config, quantiles_dir):
- """Creates a calibrated lattice estimator."""
- feature_names = [fc.name for fc in feature_columns]
- hparams = tfl.CalibratedLatticeHParams(
- feature_names=feature_names,
- num_keypoints=200,
- lattice_l2_laplacian_reg=5.0e-3,
- lattice_l2_torsion_reg=1.0e-4,
- learning_rate=0.1,
- lattice_size=2)
- hparams.parse(FLAGS.hparams)
- _pprint_hparams(hparams)
- return tfl.calibrated_lattice_classifier(
- feature_columns=feature_columns,
- model_dir=config.model_dir,
- config=config,
- hparams=hparams,
- quantiles_dir=quantiles_dir)
-
-
-def create_calibrated_rtl(feature_columns, config, quantiles_dir):
- """Creates a calibrated RTL estimator."""
- feature_names = [fc.name for fc in feature_columns]
- hparams = tfl.CalibratedRtlHParams(
- feature_names=feature_names,
- num_keypoints=200,
- learning_rate=0.02,
- lattice_l2_laplacian_reg=5.0e-4,
- lattice_l2_torsion_reg=1.0e-4,
- lattice_size=3,
- lattice_rank=4,
- num_lattices=100)
- # Specific feature parameters.
- hparams.set_feature_param("capital_gain", "lattice_size", 8)
- hparams.set_feature_param("native_country", "lattice_size", 8)
- hparams.set_feature_param("marital_status", "lattice_size", 4)
- hparams.set_feature_param("age", "lattice_size", 8)
- hparams.parse(FLAGS.hparams)
- _pprint_hparams(hparams)
- return tfl.calibrated_rtl_classifier(
- feature_columns=feature_columns,
- model_dir=config.model_dir,
- config=config,
- hparams=hparams,
- quantiles_dir=quantiles_dir)
-
-
-def create_calibrated_etl(feature_columns, config, quantiles_dir):
- """Creates a calibrated ETL estimator."""
- # No enforced monotonicity in this example.
- feature_names = [fc.name for fc in feature_columns]
- hparams = tfl.CalibratedEtlHParams(
- feature_names=feature_names,
- num_keypoints=200,
- learning_rate=0.02,
- non_monotonic_num_lattices=200,
- non_monotonic_lattice_rank=2,
- non_monotonic_lattice_size=2,
- calibration_l2_laplacian_reg=4.0e-3,
- lattice_l2_laplacian_reg=1.0e-5,
- lattice_l2_torsion_reg=4.0e-4)
- hparams.parse(FLAGS.hparams)
- _pprint_hparams(hparams)
- return tfl.calibrated_etl_classifier(
- feature_columns=feature_columns,
- model_dir=config.model_dir,
- config=config,
- hparams=hparams,
- quantiles_dir=quantiles_dir)
-
-
-def create_calibrated_dnn(feature_columns, config, quantiles_dir):
- """Creates a calibrated DNN model."""
- # This is an example of a hybrid model that uses input calibration layer
- # offered by TensorFlow Lattice library and connects it to a DNN.
- feature_names = [fc.name for fc in feature_columns]
- hparams = tfl.CalibratedHParams(
- feature_names=feature_names,
- num_keypoints=200,
- learning_rate=1.0e-3,
- calibration_output_min=-1.0,
- calibration_output_max=1.0,
- nodes_per_layer=10, # All layers have the same number of nodes.
- layers=2, # Includes output layer, therefore >= 1.
- )
- hparams.parse(FLAGS.hparams)
- _pprint_hparams(hparams)
-
- def _model_fn(features, labels, mode, params):
- """Model construction closure used when creating estimator."""
- del params # Hyper-params are read directly from the bound variable hparams
-
- # Calibrate: since there is no monotonicity, there are no projection ops.
- # We also discard the ordered names of the features.
- (output, _, _, regularization) = tfl.input_calibration_layer_from_hparams(
- features, feature_columns, hparams, quantiles_dir)
-
- # Hidden-layers.
- for _ in range(hparams.layers - 1):
- output = tf.layers.dense(
- inputs=output, units=hparams.nodes_per_layer, activation=tf.sigmoid)
-
- # Classifier logits and prediction.
- logits = tf.layers.dense(inputs=output, units=1)
- predictions = tf.reshape(tf.sigmoid(logits), [-1])
-
- # Notice loss doesn't include regularization, which is added separately
- # by means of tf.contrib.layers.apply_regularization().
- loss_no_regularization = tf.losses.log_loss(labels, predictions)
- loss = loss_no_regularization
- if regularization is not None:
- loss += regularization
- optimizer = tf.train.AdamOptimizer(learning_rate=hparams.learning_rate)
- train_op = optimizer.minimize(
- loss,
- global_step=tf.train.get_global_step(),
- name="calibrated_dnn_minimize")
-
- eval_metric_ops = {
- "accuracy": tf.metrics.accuracy(labels, predictions),
-
- # We want to report the loss without the regularization, so metric is
- # comparable with different regularizations. FutureWork, list both.
- "average_loss": tf.metrics.mean(loss_no_regularization),
- }
-
- return tf.estimator.EstimatorSpec(mode, predictions, loss, train_op,
- eval_metric_ops)
-
- # Hyper-parameters are passed directly to the model_fn closure by the context.
- return tf.estimator.Estimator(
- model_fn=_model_fn,
- model_dir=config.model_dir,
- config=config,
- params=None)
-
-
-def create_estimator(config, quantiles_dir):
- """Creates estimator for given configuration based on --model_type."""
- feature_columns = create_feature_columns()
- if FLAGS.model_type == "calibrated_linear":
- return create_calibrated_linear(feature_columns, config, quantiles_dir)
- elif FLAGS.model_type == "calibrated_lattice":
- return create_calibrated_lattice(feature_columns, config, quantiles_dir)
- elif FLAGS.model_type == "calibrated_rtl":
- return create_calibrated_rtl(feature_columns, config, quantiles_dir)
- elif FLAGS.model_type == "calibrated_etl":
- return create_calibrated_etl(feature_columns, config, quantiles_dir)
- elif FLAGS.model_type == "calibrated_dnn":
- return create_calibrated_dnn(feature_columns, config, quantiles_dir)
-
- raise ValueError("Unknown model_type={}".format(FLAGS.model_type))
-
-
-def evaluate_on_data(estimator, data):
- """Evaluates and prints results, set data to FLAGS.test or FLAGS.train."""
- name = os.path.basename(data)
- evaluation = estimator.evaluate(
- input_fn=get_input_fn(
- file_path=data,
- batch_size=FLAGS.batch_size,
- num_epochs=1,
- shuffle=False),
- name=name)
- print(" Evaluation on '{}':\taccuracy={:.4f}\taverage_loss={:.4f}".format(
- name, evaluation["accuracy"], evaluation["average_loss"]))
-
-
-def train(estimator):
- """Trains estimator and optionally intermediary evaluations."""
- if not FLAGS.train_evaluate_on_train and not FLAGS.train_evaluate_on_test:
- estimator.train(input_fn=get_train_input_fn(
- batch_size=FLAGS.batch_size,
- num_epochs=FLAGS.train_epochs,
- shuffle=True))
- else:
- # Train 1/10th of the epochs requested per loop, but at least 1 per loop.
- epochs_trained = 0
- loops = 0
- while epochs_trained < FLAGS.train_epochs:
- loops += 1
- next_epochs_trained = int(loops * FLAGS.train_epochs / 10.0)
- epochs = max(1, next_epochs_trained - epochs_trained)
- epochs_trained += epochs
- estimator.train(input_fn=get_train_input_fn(
- batch_size=FLAGS.batch_size, num_epochs=epochs, shuffle=True))
- print("Trained for {} epochs, total so far {}:".format(
- epochs, epochs_trained))
- evaluate_on_data(estimator, FLAGS.train)
- evaluate_on_data(estimator, FLAGS.test)
-
-
-def evaluate(estimator):
- """Runs straight evaluation on a currently trained model."""
- evaluate_on_data(estimator, FLAGS.train)
- evaluate_on_data(estimator, FLAGS.test)
-
-
-def main(args):
- del args # Not used.
-
- # Prepare directories.
- output_dir = FLAGS.output_dir
- if output_dir is None:
- output_dir = tempfile.mkdtemp()
- tf.logging.warning("Using temporary folder as model directory: %s",
- output_dir)
- quantiles_dir = FLAGS.quantiles_dir or output_dir
-
- # Create quantiles if required.
- if FLAGS.create_quantiles:
- if FLAGS.run != "train":
- raise ValueError(
- "Can not create_quantiles for mode --run='{}'".format(FLAGS.run))
- create_quantiles(quantiles_dir)
-
- # Create config and then model.
- config = tf.estimator.RunConfig().replace(model_dir=output_dir)
- estimator = create_estimator(config, quantiles_dir)
-
- if FLAGS.run == "train":
- train(estimator)
-
- elif FLAGS.run == "evaluate":
- evaluate(estimator)
-
- else:
- raise ValueError("Unknonw --run={}".format(FLAGS.run))
-
-
-if __name__ == "__main__":
- tf.app.run()
diff --git a/g3doc/api_docs/python/_toc.yaml b/g3doc/api_docs/python/_toc.yaml
deleted file mode 100644
index 2a3a8c1..0000000
--- a/g3doc/api_docs/python/_toc.yaml
+++ /dev/null
@@ -1,62 +0,0 @@
-# Automatically generated file; please do not edit
-toc:
- - title: tensorflow_lattice
- section:
- - title: Overview
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice
- - title: CalibratedEtlHParams
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/CalibratedEtlHParams
- - title: CalibratedHParams
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/CalibratedHParams
- - title: CalibratedLatticeHParams
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/CalibratedLatticeHParams
- - title: CalibratedLinearHParams
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/CalibratedLinearHParams
- - title: CalibratedRtlHParams
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/CalibratedRtlHParams
- - title: calibrated_etl_classifier
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/calibrated_etl_classifier
- - title: calibrated_etl_regressor
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/calibrated_etl_regressor
- - title: calibrated_lattice_classifier
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/calibrated_lattice_classifier
- - title: calibrated_lattice_regressor
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/calibrated_lattice_regressor
- - title: calibrated_linear_classifier
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/calibrated_linear_classifier
- - title: calibrated_linear_regressor
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/calibrated_linear_regressor
- - title: calibrated_rtl_classifier
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/calibrated_rtl_classifier
- - title: calibrated_rtl_regressor
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/calibrated_rtl_regressor
- - title: calibration_layer
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/calibration_layer
- - title: calibrator_regularization
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/calibrator_regularization
- - title: ensemble_lattices_layer
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/ensemble_lattices_layer
- - title: input_calibration_layer
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/input_calibration_layer
- - title: input_calibration_layer_from_hparams
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/input_calibration_layer_from_hparams
- - title: lattice
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/lattice
- - title: lattice_layer
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/lattice_layer
- - title: lattice_regularization
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/lattice_regularization
- - title: load_keypoints_from_quantiles
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/load_keypoints_from_quantiles
- - title: monotone_lattice
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/monotone_lattice
- - title: monotonic_projection
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/monotonic_projection
- - title: PerFeatureHParams
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/PerFeatureHParams
- - title: pwl_indexing_calibrator
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/pwl_indexing_calibrator
- - title: save_quantiles_for_keypoints
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/save_quantiles_for_keypoints
- - title: uniform_keypoints_for_signal
- path: /TARGET_DOC_ROOT/VERSION/api_docs/python/tensorflow_lattice/uniform_keypoints_for_signal
diff --git a/g3doc/api_docs/python/index.md b/g3doc/api_docs/python/index.md
deleted file mode 100644
index 33a4bdc..0000000
--- a/g3doc/api_docs/python/index.md
+++ /dev/null
@@ -1,31 +0,0 @@
-# All symbols in TensorFlow Lattice
-
-* [`tensorflow_lattice`](./tensorflow_lattice.md)
-* [`tensorflow_lattice.CalibratedEtlHParams`](./tensorflow_lattice/CalibratedEtlHParams.md)
-* [`tensorflow_lattice.CalibratedHParams`](./tensorflow_lattice/CalibratedHParams.md)
-* [`tensorflow_lattice.CalibratedLatticeHParams`](./tensorflow_lattice/CalibratedLatticeHParams.md)
-* [`tensorflow_lattice.CalibratedLinearHParams`](./tensorflow_lattice/CalibratedLinearHParams.md)
-* [`tensorflow_lattice.CalibratedRtlHParams`](./tensorflow_lattice/CalibratedRtlHParams.md)
-* [`tensorflow_lattice.PerFeatureHParams`](./tensorflow_lattice/PerFeatureHParams.md)
-* [`tensorflow_lattice.calibrated_etl_classifier`](./tensorflow_lattice/calibrated_etl_classifier.md)
-* [`tensorflow_lattice.calibrated_etl_regressor`](./tensorflow_lattice/calibrated_etl_regressor.md)
-* [`tensorflow_lattice.calibrated_lattice_classifier`](./tensorflow_lattice/calibrated_lattice_classifier.md)
-* [`tensorflow_lattice.calibrated_lattice_regressor`](./tensorflow_lattice/calibrated_lattice_regressor.md)
-* [`tensorflow_lattice.calibrated_linear_classifier`](./tensorflow_lattice/calibrated_linear_classifier.md)
-* [`tensorflow_lattice.calibrated_linear_regressor`](./tensorflow_lattice/calibrated_linear_regressor.md)
-* [`tensorflow_lattice.calibrated_rtl_classifier`](./tensorflow_lattice/calibrated_rtl_classifier.md)
-* [`tensorflow_lattice.calibrated_rtl_regressor`](./tensorflow_lattice/calibrated_rtl_regressor.md)
-* [`tensorflow_lattice.calibration_layer`](./tensorflow_lattice/calibration_layer.md)
-* [`tensorflow_lattice.calibrator_regularization`](./tensorflow_lattice/calibrator_regularization.md)
-* [`tensorflow_lattice.ensemble_lattices_layer`](./tensorflow_lattice/ensemble_lattices_layer.md)
-* [`tensorflow_lattice.input_calibration_layer`](./tensorflow_lattice/input_calibration_layer.md)
-* [`tensorflow_lattice.input_calibration_layer_from_hparams`](./tensorflow_lattice/input_calibration_layer_from_hparams.md)
-* [`tensorflow_lattice.lattice`](./tensorflow_lattice/lattice.md)
-* [`tensorflow_lattice.lattice_layer`](./tensorflow_lattice/lattice_layer.md)
-* [`tensorflow_lattice.lattice_regularization`](./tensorflow_lattice/lattice_regularization.md)
-* [`tensorflow_lattice.load_keypoints_from_quantiles`](./tensorflow_lattice/load_keypoints_from_quantiles.md)
-* [`tensorflow_lattice.monotone_lattice`](./tensorflow_lattice/monotone_lattice.md)
-* [`tensorflow_lattice.monotonic_projection`](./tensorflow_lattice/monotonic_projection.md)
-* [`tensorflow_lattice.pwl_indexing_calibrator`](./tensorflow_lattice/pwl_indexing_calibrator.md)
-* [`tensorflow_lattice.save_quantiles_for_keypoints`](./tensorflow_lattice/save_quantiles_for_keypoints.md)
-* [`tensorflow_lattice.uniform_keypoints_for_signal`](./tensorflow_lattice/uniform_keypoints_for_signal.md)
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice.md b/g3doc/api_docs/python/tensorflow_lattice.md
deleted file mode 100644
index 10220d3..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice.md
+++ /dev/null
@@ -1,83 +0,0 @@
-
-
-
-
-
-
-# Module: tensorflow_lattice
-
-Lattice modeling.
-
-This package provides functions and classes for lattice modeling.
-
-See full description in `README.md` file.
-
-
- use them.
-
-## Classes
-
-[`class CalibratedEtlHParams`](./tensorflow_lattice/CalibratedEtlHParams.md): Hyper-parameters for CalibratedEtl (Embedded tiny lattices) models.
-
-[`class CalibratedHParams`](./tensorflow_lattice/CalibratedHParams.md): PerFeatureHParams specialization with input calibration parameters.
-
-[`class CalibratedLatticeHParams`](./tensorflow_lattice/CalibratedLatticeHParams.md): Hyper-parameters for CalibratedLattice models.
-
-[`class CalibratedLinearHParams`](./tensorflow_lattice/CalibratedLinearHParams.md): Hyper-parameters for CalibratedLinear models.
-
-[`class CalibratedRtlHParams`](./tensorflow_lattice/CalibratedRtlHParams.md): Hyper-parameters for CalibratedRtl (RandomTinyLattices) models.
-
-[`class PerFeatureHParams`](./tensorflow_lattice/PerFeatureHParams.md): Parameters object with per feature parametrization.
-
-## Functions
-
-[`calibrated_etl_classifier(...)`](./tensorflow_lattice/calibrated_etl_classifier.md): Calibrated etl binary classifier model.
-
-[`calibrated_etl_regressor(...)`](./tensorflow_lattice/calibrated_etl_regressor.md): Calibrated etl regressor model.
-
-[`calibrated_lattice_classifier(...)`](./tensorflow_lattice/calibrated_lattice_classifier.md): Calibrated lattice classifier binary model.
-
-[`calibrated_lattice_regressor(...)`](./tensorflow_lattice/calibrated_lattice_regressor.md): Calibrated lattice estimator (model) for regression.
-
-[`calibrated_linear_classifier(...)`](./tensorflow_lattice/calibrated_linear_classifier.md): Calibrated linear classifier binary model.
-
-[`calibrated_linear_regressor(...)`](./tensorflow_lattice/calibrated_linear_regressor.md): Calibrated linear estimator (model) for regression.
-
-[`calibrated_rtl_classifier(...)`](./tensorflow_lattice/calibrated_rtl_classifier.md): Calibrated rtl binary classifier model.
-
-[`calibrated_rtl_regressor(...)`](./tensorflow_lattice/calibrated_rtl_regressor.md): Calibrated rtl regressor model.
-
-[`calibration_layer(...)`](./tensorflow_lattice/calibration_layer.md): Creates a calibration layer for uncalibrated values.
-
-[`calibrator_regularization(...)`](./tensorflow_lattice/calibrator_regularization.md): Returns a calibrator regularization op.
-
-[`ensemble_lattices_layer(...)`](./tensorflow_lattice/ensemble_lattices_layer.md): Creates a ensemble of lattices layer.
-
-[`input_calibration_layer(...)`](./tensorflow_lattice/input_calibration_layer.md): Creates a calibration layer for the given input and feature_columns.
-
-[`input_calibration_layer_from_hparams(...)`](./tensorflow_lattice/input_calibration_layer_from_hparams.md): Creates a calibration layer for the input using hyper-parameters.
-
-[`lattice(...)`](./tensorflow_lattice/lattice.md): Returns an interpolated look-up table (lattice) op.
-
-[`lattice_layer(...)`](./tensorflow_lattice/lattice_layer.md): Creates a lattice layer.
-
-[`lattice_regularization(...)`](./tensorflow_lattice/lattice_regularization.md): Returns a lattice regularization op.
-
-[`load_keypoints_from_quantiles(...)`](./tensorflow_lattice/load_keypoints_from_quantiles.md): Retrieves keypoints initialization values for selected features.
-
-[`monotone_lattice(...)`](./tensorflow_lattice/monotone_lattice.md): Returns a projected lattice parameters onto the monotonicity constraints.
-
-[`monotonic_projection(...)`](./tensorflow_lattice/monotonic_projection.md): Returns a not-strict monotonic projection of the vector.
-
-[`pwl_indexing_calibrator(...)`](./tensorflow_lattice/pwl_indexing_calibrator.md): Returns tensor representing interpolation weights in a piecewise linear
-
-[`save_quantiles_for_keypoints(...)`](./tensorflow_lattice/save_quantiles_for_keypoints.md): Calculates and saves quantiles for given features.
-
-[`uniform_keypoints_for_signal(...)`](./tensorflow_lattice/uniform_keypoints_for_signal.md): Returns a pair of initialization tensors for calibration keypoints.
-
-## Other Members
-
-`DEFAULT_NAME`
-
-`absolute_import`
-
diff --git a/g3doc/api_docs/python/tensorflow_lattice/CalibratedEtlHParams.md b/g3doc/api_docs/python/tensorflow_lattice/CalibratedEtlHParams.md
deleted file mode 100644
index 73d4cc0..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/CalibratedEtlHParams.md
+++ /dev/null
@@ -1,294 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-# tensorflow_lattice.CalibratedEtlHParams
-
-## Class `CalibratedEtlHParams`
-
-Inherits From: [`CalibratedHParams`](../tensorflow_lattice/CalibratedHParams.md)
-
-Hyper-parameters for CalibratedEtl (Embedded tiny lattices) models.
-
-Supports regularization and monotonicity like described in `CalibratedHParam`.
-Values for `calibration_output_min`, `calibration_output_max` and
-`missing_output_value` get set automatically.
-
-Note that this architecture does not support any of per-feature based lattice
-hyper-parameters such as missing_vertex, per-feature missing_input_value,
-per-feature lattice_size, per-feature lattice regularization, because after
-the linear embedding, all of features are mixed together, so it is not clear
-how to merge per-feature parameters after the linear embedding layer.
-
-If there is no non-monotonic feature, but `non_monotonic_lattice_rank` or
-`non_monotonic_num_lattices` are not `None`, then this will raise the error.
-
-Added parameters:
-
-* `learning_rate`: (float) a global parameter that assigns a step size of an
- optimizer.
-* `lattice_size`: (int) a global parameter that controls number of
- cells for a feature. Should be greater than equal to 2, and the recommended
- default value is 2. Also calibrator output min and max should be
- [0, `lattice_size` - 1], and the output should be bounded.
-* `interpolation_type`: a global parameter that defines if the lattice will
- interpolate using the full hypercube or only the simplex ("hyper-triangle",
- much faster for larger lattices) around the point being evaluated.
- Valid values: 'hypercube' or 'simplex'
-* `monotonic_lattice_rank`: (int) a lattice rank in each monotonic lattice.
-* `monotonic_num_lattices`: (int) a number of monotonic lattices to be
- created.
-* `monotonic_lattice_size`: (int) lattice cell size for each monotonic lattice
- in the ensemble lattices layer.
-* `non_monotonic_lattice_rank`: (int) a lattice rank in each non monotonic
- lattice. If all features are monotonic, this parameter should be None.
-* `non_monotonic_num_lattices`: (int) a number of non-monotonic lattices to be
- created. If all features are monotonic, this parameter should be None.
-* `monotonic_lattice_size`: (int) lattice cell size for each non-monotonic
- lattice in the ensemble lattices layer.
-* `linear_embedding_calibration_min`: (float) a global parameter that controls
- a minimum value of intermediate calibration layers. Default is -100.
-* `linear_embedding_calibration_max`: (float) a global parameter that controls
- a maximum value of intermediate calibration layers. Default is 100.
-* `linear_embedding_calibration_num_keypoints`: (float) a global parameter
- that controls a `num_keypoints` in intermediate calibration layers. Default
- is 100.
-
-## Methods
-
-
-
-``` python
-add_feature(feature_name)
-```
-
-Add feature_name (one name or list of names) to list of known names.
-
-
get_feature_names
-
-``` python
-get_feature_names()
-```
-
-Returns copy of list of known feature names.
-
-
get_feature_param
-
-``` python
-get_feature_param(
- feature_name,
- param_name,
- default=None
-)
-```
-
-Returns parameter for feature or falls back to global parameter.
-
-
get_global_and_feature_params
-
-``` python
-get_global_and_feature_params(
- param_names,
- feature_names
-)
-```
-
-Returns values for multiple params, global and for each feature.
-
-#### Args:
-
-* `param_names`: list of parameters to get values for.
-* `feature_names`: list of features to get specific values for.
-
-
-#### Returns:
-
-* List of global values for parameters requested in `param_names`.
-* List of list of per feature values for parameters requested in
- `param_names` for features requested in `feature_names`.
-
-
get_param
-
-``` python
-get_param(
- param_name,
- default=None
-)
-```
-
-Returns the global parameter or falls back to default.
-
-
is_feature_set_param
-
-``` python
-is_feature_set_param(
- feature_name,
- param_name
-)
-```
-
-Returns whether param_name parameter is set for feature_name.
-
-
param_name_for_feature
-
-``` python
-param_name_for_feature(
- feature_name,
- param_name
-)
-```
-
-Returns parameter name for specific feature parameter.
-
-
parse
-
-``` python
-parse(hparams_str)
-```
-
-Parses strings into hparams.
-
-#### Args:
-
-* `hparams_str`: must be a comma separated list of "=",
- where "" is a hyper-parameter name, and "" its value.
-
-
-#### Returns:
-
-Changes affect self, but returns self for convenience.
-
-
-#### Raises:
-
-* `ValueError`: if there is a problem with the input:
- * if trying to set an unknown parameter.
- * if trying to set unknown feature(s)
- * if can't convert value to parameter type.
-
-
parse_hparams
-
-``` python
-parse_hparams(hparams)
-```
-
-Incorporates hyper-parameters from another HParams object.
-
-Copies over values of hyper-parameters from the given object. New parameters
-may be set, but not new features. Also works with
-`tf.contrib.training.HParams` objects.
-
-#### Args:
-
-* `hparams`: `PerFeatureHParams` object, but also works with the standard
- `tf.contrib.training.HParams` object.
-
-
-#### Returns:
-
-Changes affect self, but returns self for convenience.
-
-
-#### Raises:
-
-* `ValueError`: if trying to set unknown features, or if setting a feature
- specific parameter for an unknown parameter.
-
-
-
-# tensorflow_lattice.CalibratedHParams
-
-## Class `CalibratedHParams`
-
-Inherits From: [`PerFeatureHParams`](../tensorflow_lattice/PerFeatureHParams.md)
-
-PerFeatureHParams specialization with input calibration parameters.
-
-The following hyper-parameters can be set as global, or per-feature (see
-base `PerFeatureHParams` for details):
-
- * `feature_names`: list of feature names. Only features names listed here
- (or added later with add_feature) can have feature specific parameter
- values.
- * `num_keypoints`: Number of keypoints to use for calibration, Set to 0 or
- `None` for no calibration.
- * `calibration_output_min`, `calibration_output_max`: initial and final
- values for calibrations. -1.0 to 1.0 works well for calibrated linear
- models. For lattices one will want to set these to (0, `lattice_size`-1).
- Only used during initialization of the calibration, if `quantiles_dir`
- is given to the calibrated model (as opposed to defining one's own value
- with `keypoints_initializers_fn`). It must be defined for calibration to
- work, no default is set.
- * `calibration_bound`: If output of calibration max/min are bound to the
- limits given in `calibration_output_min/max`.
- * `monotonicity`: Monotonicity for the feature. 0 for no monotonicity,
- 1 and -1 for increasing and decreasing monotonicity respectively.
- * `missing_input_value`: If set, and if the input has this value it is
- assumed
- to be missing and the output will either be calibrated to some value
- between `[calibration_output_min, calibration_output_max]` or set to a
- fixed value set by missing_output_value.
- * `missing_output_value`: Requires missing_input_value also to be set. If
- set
- if will convert missing input to this value. Leave it undefined and the
- output will be learned.
- * `calibration_l1_reg`, `calibration_l2_reg`,
- `calibration_l1_laplacian_reg`, `calibration_l2_laplacian_reg`: Calibrator
- regularizers regularization amount. Default is `None`.
-
-## Methods
-
-
-
-``` python
-add_feature(feature_name)
-```
-
-Add feature_name (one name or list of names) to list of known names.
-
-
get_feature_names
-
-``` python
-get_feature_names()
-```
-
-Returns copy of list of known feature names.
-
-
get_feature_param
-
-``` python
-get_feature_param(
- feature_name,
- param_name,
- default=None
-)
-```
-
-Returns parameter for feature or falls back to global parameter.
-
-
get_global_and_feature_params
-
-``` python
-get_global_and_feature_params(
- param_names,
- feature_names
-)
-```
-
-Returns values for multiple params, global and for each feature.
-
-#### Args:
-
-* `param_names`: list of parameters to get values for.
-* `feature_names`: list of features to get specific values for.
-
-
-#### Returns:
-
-* List of global values for parameters requested in `param_names`.
-* List of list of per feature values for parameters requested in
- `param_names` for features requested in `feature_names`.
-
-
get_param
-
-``` python
-get_param(
- param_name,
- default=None
-)
-```
-
-Returns the global parameter or falls back to default.
-
-
is_feature_set_param
-
-``` python
-is_feature_set_param(
- feature_name,
- param_name
-)
-```
-
-Returns whether param_name parameter is set for feature_name.
-
-
param_name_for_feature
-
-``` python
-param_name_for_feature(
- feature_name,
- param_name
-)
-```
-
-Returns parameter name for specific feature parameter.
-
-
parse
-
-``` python
-parse(hparams_str)
-```
-
-Parses strings into hparams.
-
-#### Args:
-
-* `hparams_str`: must be a comma separated list of "=",
- where "" is a hyper-parameter name, and "" its value.
-
-
-#### Returns:
-
-Changes affect self, but returns self for convenience.
-
-
-#### Raises:
-
-* `ValueError`: if there is a problem with the input:
- * if trying to set an unknown parameter.
- * if trying to set unknown feature(s)
- * if can't convert value to parameter type.
-
-
parse_hparams
-
-``` python
-parse_hparams(hparams)
-```
-
-Incorporates hyper-parameters from another HParams object.
-
-Copies over values of hyper-parameters from the given object. New parameters
-may be set, but not new features. Also works with
-`tf.contrib.training.HParams` objects.
-
-#### Args:
-
-* `hparams`: `PerFeatureHParams` object, but also works with the standard
- `tf.contrib.training.HParams` object.
-
-
-#### Returns:
-
-Changes affect self, but returns self for convenience.
-
-
-#### Raises:
-
-* `ValueError`: if trying to set unknown features, or if setting a feature
- specific parameter for an unknown parameter.
-
-
-
-# tensorflow_lattice.CalibratedLatticeHParams
-
-## Class `CalibratedLatticeHParams`
-
-Inherits From: [`CalibratedHParams`](../tensorflow_lattice/CalibratedHParams.md)
-
-Hyper-parameters for CalibratedLattice models.
-
-Supports regularization and monotonicity like described in `CalibratedHParam`.
-Values for `calibration_output_min`, `calibration_output_max` and
-`missing_output_value` get set automatically.
-
-Added parameters:
-
-* `learning_rate`: (float) a global parameter that assigns a step size of an
- optimizer.
-* `lattice_size`: (int) a global or per feature parameter that controls number
- of cells for a feature. Should be greater than equal to 2, and the
- recommended default value is 2. Also calibrator output min and max should be
- [0, lattice_size - 1], and the output should be bounded, since a lattice
- expects an input in the range [0, lattice_size - 1].
-* `interpolation_type`: a global parameter that defines if the lattice will
- interpolate using the full hypercube or only the simplex ("hyper-triangle",
- much faster for larger lattices) around the point being evaluated.
- Valid values: 'hypercube' or 'simplex'
-* `missing_input_value`: Value for which a feature is considered missing. Such
- values are either automatically learned to some calibrated value, or,
- if missing_vertex is set, they get their own value in the lattice.
-* `missing_vertex`: if missing_input_value is set, this boolean value indicate
- whether to create an extra vertex for missing values.
-* `lattice_l1_reg`, `lattice_l2_reg`, `lattice_l1_torsion_reg`,
- `lattice_l2_torsion_reg`, `lattice_l1_laplacian_reg`,
- `lattice_l2_laplacian_reg`: Lattice regularizers regularization amount.
- Default is `None`.
-
-## Methods
-
-
-
-``` python
-add_feature(feature_name)
-```
-
-Add feature_name (one name or list of names) to list of known names.
-
-
get_feature_names
-
-``` python
-get_feature_names()
-```
-
-Returns copy of list of known feature names.
-
-
get_feature_param
-
-``` python
-get_feature_param(
- feature_name,
- param_name,
- default=None
-)
-```
-
-Returns parameter for feature or falls back to global parameter.
-
-
get_global_and_feature_params
-
-``` python
-get_global_and_feature_params(
- param_names,
- feature_names
-)
-```
-
-Returns values for multiple params, global and for each feature.
-
-#### Args:
-
-* `param_names`: list of parameters to get values for.
-* `feature_names`: list of features to get specific values for.
-
-
-#### Returns:
-
-* List of global values for parameters requested in `param_names`.
-* List of list of per feature values for parameters requested in
- `param_names` for features requested in `feature_names`.
-
-
get_param
-
-``` python
-get_param(
- param_name,
- default=None
-)
-```
-
-Returns the global parameter or falls back to default.
-
-
is_feature_set_param
-
-``` python
-is_feature_set_param(
- feature_name,
- param_name
-)
-```
-
-Returns whether param_name parameter is set for feature_name.
-
-
param_name_for_feature
-
-``` python
-param_name_for_feature(
- feature_name,
- param_name
-)
-```
-
-Returns parameter name for specific feature parameter.
-
-
parse
-
-``` python
-parse(hparams_str)
-```
-
-Parses strings into hparams.
-
-#### Args:
-
-* `hparams_str`: must be a comma separated list of "=",
- where "" is a hyper-parameter name, and "" its value.
-
-
-#### Returns:
-
-Changes affect self, but returns self for convenience.
-
-
-#### Raises:
-
-* `ValueError`: if there is a problem with the input:
- * if trying to set an unknown parameter.
- * if trying to set unknown feature(s)
- * if can't convert value to parameter type.
-
-
parse_hparams
-
-``` python
-parse_hparams(hparams)
-```
-
-Incorporates hyper-parameters from another HParams object.
-
-Copies over values of hyper-parameters from the given object. New parameters
-may be set, but not new features. Also works with
-`tf.contrib.training.HParams` objects.
-
-#### Args:
-
-* `hparams`: `PerFeatureHParams` object, but also works with the standard
- `tf.contrib.training.HParams` object.
-
-
-#### Returns:
-
-Changes affect self, but returns self for convenience.
-
-
-#### Raises:
-
-* `ValueError`: if trying to set unknown features, or if setting a feature
- specific parameter for an unknown parameter.
-
-
-
-# tensorflow_lattice.CalibratedLinearHParams
-
-## Class `CalibratedLinearHParams`
-
-Inherits From: [`CalibratedHParams`](../tensorflow_lattice/CalibratedHParams.md)
-
-Hyper-parameters for CalibratedLinear models.
-
-Same as `CalibratedHParams` (hyper-parameters for input calibration) plus
-the global learning_rate.
-
-The parameters `calibration_output_min` and `calibration_output_max` shouldn't
-be changed (they are fixed at -1. and +1), since they are eventually re-scaled
-by the linear layer on top.
-
-It supports regularization, monotonicity and missing values (input and
-optionally output).
-
-## Methods
-
-
-
-``` python
-add_feature(feature_name)
-```
-
-Add feature_name (one name or list of names) to list of known names.
-
-
get_feature_names
-
-``` python
-get_feature_names()
-```
-
-Returns copy of list of known feature names.
-
-
get_feature_param
-
-``` python
-get_feature_param(
- feature_name,
- param_name,
- default=None
-)
-```
-
-Returns parameter for feature or falls back to global parameter.
-
-
get_global_and_feature_params
-
-``` python
-get_global_and_feature_params(
- param_names,
- feature_names
-)
-```
-
-Returns values for multiple params, global and for each feature.
-
-#### Args:
-
-* `param_names`: list of parameters to get values for.
-* `feature_names`: list of features to get specific values for.
-
-
-#### Returns:
-
-* List of global values for parameters requested in `param_names`.
-* List of list of per feature values for parameters requested in
- `param_names` for features requested in `feature_names`.
-
-
get_param
-
-``` python
-get_param(
- param_name,
- default=None
-)
-```
-
-Returns the global parameter or falls back to default.
-
-
is_feature_set_param
-
-``` python
-is_feature_set_param(
- feature_name,
- param_name
-)
-```
-
-Returns whether param_name parameter is set for feature_name.
-
-
param_name_for_feature
-
-``` python
-param_name_for_feature(
- feature_name,
- param_name
-)
-```
-
-Returns parameter name for specific feature parameter.
-
-
parse
-
-``` python
-parse(hparams_str)
-```
-
-Parses strings into hparams.
-
-#### Args:
-
-* `hparams_str`: must be a comma separated list of "=",
- where "" is a hyper-parameter name, and "" its value.
-
-
-#### Returns:
-
-Changes affect self, but returns self for convenience.
-
-
-#### Raises:
-
-* `ValueError`: if there is a problem with the input:
- * if trying to set an unknown parameter.
- * if trying to set unknown feature(s)
- * if can't convert value to parameter type.
-
-
parse_hparams
-
-``` python
-parse_hparams(hparams)
-```
-
-Incorporates hyper-parameters from another HParams object.
-
-Copies over values of hyper-parameters from the given object. New parameters
-may be set, but not new features. Also works with
-`tf.contrib.training.HParams` objects.
-
-#### Args:
-
-* `hparams`: `PerFeatureHParams` object, but also works with the standard
- `tf.contrib.training.HParams` object.
-
-
-#### Returns:
-
-Changes affect self, but returns self for convenience.
-
-
-#### Raises:
-
-* `ValueError`: if trying to set unknown features, or if setting a feature
- specific parameter for an unknown parameter.
-
-
-
-# tensorflow_lattice.CalibratedRtlHParams
-
-## Class `CalibratedRtlHParams`
-
-Inherits From: [`CalibratedHParams`](../tensorflow_lattice/CalibratedHParams.md)
-
-Hyper-parameters for CalibratedRtl (RandomTinyLattices) models.
-
-Supports regularization and monotonicity like described in `CalibratedHParam`.
-Values for `calibration_output_min`, `calibration_output_max` and
-`missing_output_value` get set automatically.
-
-Added parameters:
-
-* `learning_rate`: (float) a global parameter that assigns a step size of an
- optimizer.
-* `lattice_size`: (int) a global or per feature parameter that controls number
- of cells for a feature. Should be greater than equal to 2, and the
- recommended default value is 2. Also calibrator output min and max should be
- [0, lattice_size - 1], and the output should be bounded, since a lattice
- expects an input in the range [0, lattice_size - 1]. (Note if missing_vertex
- is True, then we add an extra vertex, so input range is [0, lattice_size])
-* `num_lattices`: (int) a number of lattices to be created.
-* `lattice_rank`: (int) a lattice rank in each lattice.
-* `interpolation_type`: a global parameter that defines if the lattice will
- interpolate using the full hypercube or only the simplex ("hyper-triangle",
- much faster for larger lattices) around the point being evaluated.
- Valid values: 'hypercube' or 'simplex'
-* `ensemble_bias`: (float) an initial value of bias term to be added to the
- output of ensemble.
-* `rtl_seed`: (int) a random seed for rtl construction.
-* `missing_input_value`: Value for which a feature is considered missing. Such
- values are either automatically learned to some calibrated value, or,
- if missing_vertex is set, they get their own value in the lattice.
-* `missing_vertex`: if missing_input_value is set, this boolean value indicate
- whether to create an extra vertex for missing values.
-* `lattice_l1_reg`, `lattice_l2_reg`, `lattice_l1_torsion_reg`,
- `lattice_l2_torsion_reg`, `lattice_l1_laplacian_reg`,
- `lattice_l2_laplacian_reg`: Latticer regularizers regularization amount.
- Default is `None`.
-
-## Methods
-
-
-
-``` python
-add_feature(feature_name)
-```
-
-Add feature_name (one name or list of names) to list of known names.
-
-
get_feature_names
-
-``` python
-get_feature_names()
-```
-
-Returns copy of list of known feature names.
-
-
get_feature_param
-
-``` python
-get_feature_param(
- feature_name,
- param_name,
- default=None
-)
-```
-
-Returns parameter for feature or falls back to global parameter.
-
-
get_global_and_feature_params
-
-``` python
-get_global_and_feature_params(
- param_names,
- feature_names
-)
-```
-
-Returns values for multiple params, global and for each feature.
-
-#### Args:
-
-* `param_names`: list of parameters to get values for.
-* `feature_names`: list of features to get specific values for.
-
-
-#### Returns:
-
-* List of global values for parameters requested in `param_names`.
-* List of list of per feature values for parameters requested in
- `param_names` for features requested in `feature_names`.
-
-
get_param
-
-``` python
-get_param(
- param_name,
- default=None
-)
-```
-
-Returns the global parameter or falls back to default.
-
-
is_feature_set_param
-
-``` python
-is_feature_set_param(
- feature_name,
- param_name
-)
-```
-
-Returns whether param_name parameter is set for feature_name.
-
-
param_name_for_feature
-
-``` python
-param_name_for_feature(
- feature_name,
- param_name
-)
-```
-
-Returns parameter name for specific feature parameter.
-
-
parse
-
-``` python
-parse(hparams_str)
-```
-
-Parses strings into hparams.
-
-#### Args:
-
-* `hparams_str`: must be a comma separated list of "=",
- where "" is a hyper-parameter name, and "" its value.
-
-
-#### Returns:
-
-Changes affect self, but returns self for convenience.
-
-
-#### Raises:
-
-* `ValueError`: if there is a problem with the input:
- * if trying to set an unknown parameter.
- * if trying to set unknown feature(s)
- * if can't convert value to parameter type.
-
-
parse_hparams
-
-``` python
-parse_hparams(hparams)
-```
-
-Incorporates hyper-parameters from another HParams object.
-
-Copies over values of hyper-parameters from the given object. New parameters
-may be set, but not new features. Also works with
-`tf.contrib.training.HParams` objects.
-
-#### Args:
-
-* `hparams`: `PerFeatureHParams` object, but also works with the standard
- `tf.contrib.training.HParams` object.
-
-
-#### Returns:
-
-Changes affect self, but returns self for convenience.
-
-
-#### Raises:
-
-* `ValueError`: if trying to set unknown features, or if setting a feature
- specific parameter for an unknown parameter.
-
-
-
-# tensorflow_lattice.PerFeatureHParams
-
-## Class `PerFeatureHParams`
-
-
-
-Parameters object with per feature parametrization.
-
-Each parameter can be overwritten for specific features by setting
-`feature____`, otherwise it falls back to the
-global parameter name value ``.
-
-Parameter types are set from their first value set -- but they can also be
-reset by `set_param_type`.
-
-Example: let's say we have a parameter `lattice_size` that should be 2 if not
-specified (global value), but can be overridden per feature; let's assume
-there are 3 features: `a`, `b`, and `c` (added after construction). Then:
-
-```python
- hparams = PerFeatureHParams(["a", "b"], lattice_size=2,
- feature__b__lattice_size=3)
- hparams.add_feature(["c"])
- hparams.get_param("lattice_size") == 2
- hparams.get_feature_param("a", "lattice_size") == 2
- hparams.get_feature_param("b", "lattice_size") == 3
- hparams.get_feature_param("c", "lattice_size") == 2
- hparams.get_feature_param("d", "lattice_size") raises a ValueError
-```
-
-Use the `get_feature_param` method to automatically get the specialized value,
-or fall-back to the global one.
-
-
-
-
-
-## Methods
-
-
__init__
-
-``` python
-__init__(
- feature_names=None,
- **kwargs
-)
-```
-
-Construct with arbitrary list of parameters.
-
-#### Args:
-
-* `feature_names`: list of feature names. Only features names listed here
- (or added later with add_feature) can have feature specific parameter
- values.
-* `**kwargs`: parameters names.
-
-
-#### Returns:
-
-PerFeatureHParams object.
-
-
-#### Raises:
-
-* `ValueError`: if a feature-specific parameter value is set for an
- unknown feature.
-
-
-
-``` python
-add_feature(feature_name)
-```
-
-Add feature_name (one name or list of names) to list of known names.
-
-
get_feature_names
-
-``` python
-get_feature_names()
-```
-
-Returns copy of list of known feature names.
-
-
get_feature_param
-
-``` python
-get_feature_param(
- feature_name,
- param_name,
- default=None
-)
-```
-
-Returns parameter for feature or falls back to global parameter.
-
-
get_global_and_feature_params
-
-``` python
-get_global_and_feature_params(
- param_names,
- feature_names
-)
-```
-
-Returns values for multiple params, global and for each feature.
-
-#### Args:
-
-* `param_names`: list of parameters to get values for.
-* `feature_names`: list of features to get specific values for.
-
-
-#### Returns:
-
-* List of global values for parameters requested in `param_names`.
-* List of list of per feature values for parameters requested in
- `param_names` for features requested in `feature_names`.
-
-
get_param
-
-``` python
-get_param(
- param_name,
- default=None
-)
-```
-
-Returns the global parameter or falls back to default.
-
-
is_feature_set_param
-
-``` python
-is_feature_set_param(
- feature_name,
- param_name
-)
-```
-
-Returns whether param_name parameter is set for feature_name.
-
-
param_name_for_feature
-
-``` python
-param_name_for_feature(
- feature_name,
- param_name
-)
-```
-
-Returns parameter name for specific feature parameter.
-
-
parse
-
-``` python
-parse(hparams_str)
-```
-
-Parses strings into hparams.
-
-#### Args:
-
-* `hparams_str`: must be a comma separated list of "=",
- where "" is a hyper-parameter name, and "" its value.
-
-
-#### Returns:
-
-Changes affect self, but returns self for convenience.
-
-
-#### Raises:
-
-* `ValueError`: if there is a problem with the input:
- * if trying to set an unknown parameter.
- * if trying to set unknown feature(s)
- * if can't convert value to parameter type.
-
-
parse_hparams
-
-``` python
-parse_hparams(hparams)
-```
-
-Incorporates hyper-parameters from another HParams object.
-
-Copies over values of hyper-parameters from the given object. New parameters
-may be set, but not new features. Also works with
-`tf.contrib.training.HParams` objects.
-
-#### Args:
-
-* `hparams`: `PerFeatureHParams` object, but also works with the standard
- `tf.contrib.training.HParams` object.
-
-
-#### Returns:
-
-Changes affect self, but returns self for convenience.
-
-
-#### Raises:
-
-* `ValueError`: if trying to set unknown features, or if setting a feature
- specific parameter for an unknown parameter.
-
-
-
-# tensorflow_lattice.calibrated_etl_classifier
-
-``` python
-calibrated_etl_classifier(
- feature_columns=None,
- model_dir=None,
- quantiles_dir=None,
- keypoints_initializers_fn=None,
- optimizer=None,
- config=None,
- hparams=None
-)
-```
-
-Calibrated etl binary classifier model.
-
-
-
-This model uses a piecewise lattice calibration function on each of the
-inputs (parametrized) and then feeds them to ensemble of random lattices.
-num_lattices and lattice_rank (number of inputs to each lattice) must be
-specified in the hyperparameter. Optionally calibration can be made monotonic.
-
-It usually requires a preprocessing step on the data, to calculate the
-quantiles of each used feature. This can be done locally or in one worker
-only before training, in a separate invocation of your program (or directly).
-Typically this can be saved (`save_dir` parameter) to the same
-directory where the data is.
-
-Hyper-parameters are given in the form of the object
-tfl_hparams.CalibrationEtlHParams. lattice_rank and num_lattices must
-be specified; there would be no default value for this. It also takes in
-per-feature parameters.
-
-Internally values will be converted to tf.float32.
-
-Example:
-
-```python
-def input_fn_train: ...
-def input_fn_eval: ...
-
-my_feature_columns=[...]
-
-# Have a separate program flag to generate the quantiles. Need to be run
-# only once.
-if FLAGS.create_quantiles:
- pwl_calibrators_layers.calculate_quantiles_for_keypoints(
- input_fn=input_fn_train,
- feature_columns=my_feature_columns,
- save_dir=FLAGS.data_dir,
- num_quantiles=1000,
- override=True)
- return # Exit program.
-
-hparams = hparams.CalibratedEtlHparams(num_lattices=10, lattice_rank=2)
-estimator = calibrated_etl.calibrated_etl_classifier(
- feature_columns=feature_columns, hparams=hparams)
-estimator.train(input_fn=input_fn_train)
-estimator.evaluate(input_fn=input_fn_eval)
-estimator.predict(input_fn=input_fn_predict)
-```
-
-#### Args:
-
-* `feature_columns`: Optional, an iteratable containing all the feature
- columns used by the model. All items in the set should be instances of
- classes derived from `FeatureColumn`. If not given, the model will
- use as features the tensors returned by input_fn.
- Supported types of columns: RealValuedColumn.
-* `model_dir`: Directory to save model parameters, graphs and etc. This can
- also be used to load checkpoints from the directory into an estimator to
- continue training a previously saved model.
-* `quantiles_dir`: location where quantiles for the data was saved. Typically
- the same directory as the training data. These quantiles can be
- generated only once with
- `pwl_calibration_layers.calculate_quantiles_for_keypoints` in a separate
- invocation of your program. If you don't want to use quantiles, you can
- set `keypoints_initializer` instead.
-* `keypoints_initializers_fn`: if you know the distribution of your
- input features you can provide that directly instead of `quantiles_dir`.
- See `pwl_calibrators_layers.uniform_keypoints_for_signal`. It must be
- a closure that returns a pair of tensors with keypoints inputs and
- outputs to use for initialization (must match `num_keypoints` configured
- in `hparams`). Alternatively the closure can return a dict mapping
- feature name to pairs for initialization per feature. If `quantiles_dir`
- and `keypoints_initializers_fn` are set, the later takes precendence,
- and the features for which `keypoints_initializers` are not defined
- fallback to using the quantiles found in `quantiles_dir`. It uses a
- closure instead of the tensors themselves because the graph has to be
- created at the time the model is being build, which happens at a later
- time.
-* `optimizer`: string, `Optimizer` object, or callable that defines the
- optimizer to use for training -- if a callable, it will be called with
- learning_rate=hparams.learning_rate.
-* `config`: RunConfig object to configure the runtime settings. Typically set
- to learn_runner.EstimatorConfig().
-* `hparams`: an instance of tfl_hparams.CalibrationEtlHParams. If set to
- None default parameters are used.
-
-
-#### Returns:
-
-A `calibrated_etl_classifier` estimator.
-
-
-#### Raises:
-
-* `ValueError`: invalid parameters.
-* `KeyError`: type of feature not supported.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/calibrated_etl_regressor.md b/g3doc/api_docs/python/tensorflow_lattice/calibrated_etl_regressor.md
deleted file mode 100644
index 2c200ea..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/calibrated_etl_regressor.md
+++ /dev/null
@@ -1,112 +0,0 @@
-
-
-
-
-# tensorflow_lattice.calibrated_etl_regressor
-
-``` python
-calibrated_etl_regressor(
- feature_columns=None,
- model_dir=None,
- quantiles_dir=None,
- keypoints_initializers_fn=None,
- optimizer=None,
- config=None,
- hparams=None
-)
-```
-
-Calibrated etl regressor model.
-
-This model uses a piecewise lattice calibration function on each of the
-inputs (parametrized) and then feeds them to ensemble of random lattices.
-num_lattices and lattice_rank (number of inputs to each lattice) must be
-specified in the hyperparameter. Optionally calibration can be made monotonic.
-
-It usually requires a preprocessing step on the data, to calculate the
-quantiles of each used feature. This can be done locally or in one worker
-only before training, in a separate invocation of your program (or directly).
-Typically this can be saved (`save_dir` parameter) to the same
-directory where the data is.
-
-Hyper-parameters are given in the form of the object
-tfl_hparams.CalibrationEtlHParams. lattice_rank and num_lattices must
-be specified; there would be no default value for this. It also takes in
-per-feature parameters.
-
-Internally values will be converted to tf.float32.
-
-Example:
-
-```python
-def input_fn_train: ...
-def input_fn_eval: ...
-
-my_feature_columns=[...]
-
-# Have a separate program flag to generate the quantiles. Need to be run
-# only once.
-if FLAGS.create_quantiles:
- pwl_calibrators_layers.calculate_quantiles_for_keypoints(
- input_fn=input_fn_train,
- feature_columns=my_feature_columns,
- save_dir=FLAGS.data_dir,
- num_quantiles=1000,
- override=True)
- return # Exit program.
-
-hparams = hparams.CalibratedEtlHparams(num_lattices=10, lattice_rank=2)
-estimator = calibrated_etl.calibrated_etl_classifier(
- feature_columns=feature_columns, hparams=hparams)
-estimator.train(input_fn=input_fn_train)
-estimator.evaluate(input_fn=input_fn_eval)
-estimator.predict(input_fn=input_fn_predict)
-```
-
-#### Args:
-
-* `feature_columns`: Optional, an iteratable containing all the feature
- columns used by the model. All items in the set should be instances of
- classes derived from `FeatureColumn`. If not given, the model will
- use as features the tensors returned by input_fn.
- Supported types of columns: RealValuedColumn.
-* `model_dir`: Directory to save model parameters, graphs and etc. This can
- also be used to load checkpoints from the directory into an estimator to
- continue training a previously saved model.
-* `quantiles_dir`: location where quantiles for the data was saved. Typically
- the same directory as the training data. These quantiles can be
- generated only once with
- `pwl_calibration_layers.calculate_quantiles_for_keypoints` in a separate
- invocation of your program. If you don't want to use quantiles, you can
- set `keypoints_initializer` instead.
-* `keypoints_initializers_fn`: if you know the distribution of your
- input features you can provide that directly instead of `quantiles_dir`.
- See `pwl_calibrators_layers.uniform_keypoints_for_signal`. It must be
- a closure that returns a pair of tensors with keypoints inputs and
- outputs to use for initialization (must match `num_keypoints` configured
- in `hparams`). Alternatively the closure can return a dict mapping
- feature name to pairs for initialization per feature. If `quantiles_dir`
- and `keypoints_initializers_fn` are set, the later takes precendence,
- and the features for which `keypoints_initializers` are not defined
- fallback to using the quantiles found in `quantiles_dir`. It uses a
- closure instead of the tensors themselves because the graph has to be
- created at the time the model is being build, which happens at a later
- time.
-* `optimizer`: string, `Optimizer` object, or callable that defines the
- optimizer to use for training -- if a callable, it will be called with
- learning_rate=hparams.learning_rate.
-* `config`: RunConfig object to configure the runtime settings. Typically set
- to learn_runner.EstimatorConfig().
-* `hparams`: an instance of tfl_hparams.CalibrationEtlHParams. If set to
- None default parameters are used.
-
-
-#### Returns:
-
-A `calibrated_etl_regressor` estimator.
-
-
-#### Raises:
-
-* `ValueError`: invalid parameters.
-* `KeyError`: type of feature not supported.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/calibrated_lattice_classifier.md b/g3doc/api_docs/python/tensorflow_lattice/calibrated_lattice_classifier.md
deleted file mode 100644
index 6e0dda2..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/calibrated_lattice_classifier.md
+++ /dev/null
@@ -1,107 +0,0 @@
-
-
-
-
-# tensorflow_lattice.calibrated_lattice_classifier
-
-``` python
-calibrated_lattice_classifier(
- feature_columns=None,
- model_dir=None,
- quantiles_dir=None,
- keypoints_initializers_fn=None,
- optimizer=None,
- config=None,
- hparams=None
-)
-```
-
-Calibrated lattice classifier binary model.
-
-
-
-This model uses a piecewise lattice calibration function on each of the
-real (as opposed to binary) inputs (parametrized) and then combines (sum up)
-the results. Optionally calibration can be made monotonic.
-
-It usually requires a preprocessing step on the data, to calculate the
-quantiles of each used feature. This can be done locally or in one worker
-only before training, in a separate invocation of your program (or directly).
-Typically this can be save (`save_dir` parameter) to the same
-directory where the data is.
-
-Hyper-parameters are given in the form of the object
-tfl_hparams.CalibrationHParams. It takes in per-feature calibration
-parameters.
-
-Internally values will be converted to tf.float32.
-
-Example:
-
-```python
-def input_fn_train: ...
-def input_fn_eval: ...
-
-my_feature_columns=[...]
-
-# Have a separate program flag to generate the quantiles. Need to be run
-# only once.
-if FLAGS.create_quantiles:
- pwl_calibrators_layers.calculate_quantiles_for_keypoints(
- input_fn=input_fn_train,
- feature_columns=my_feature_columns,
- save_dir=FLAGS.data_dir,
- num_quantiles=1000,
- override=True)
- return # Exit program.
-
-estimator = calibrated_lattice.CalibratedLatticeClassifier(
- feature_columns=feature_columns)
-estimator.train(input_fn=input_fn_train)
-estimator.evaluate(input_fn=input_fn_eval)
-estimator.predict(input_fn=input_fn_predict)
-```
-
-#### Args:
-
-* `feature_columns`: Optional, an iteratable containing all the feature
- columns used by the model. All items in the set should be instances of
- classes derived from `FeatureColumn`. If not given, the model will
- use as features the tensors returned by input_fn.
- Supported types of columns: RealValuedColumn.
-* `model_dir`: Directory to save model parameters, graph and etc. This can
- also be used to load checkpoints from the directory into a estimator to
- continue training a previously saved model.
-* `quantiles_dir`: location where quantiles for the data was saved. Typically
- the same directory as the training data. These quantiles can be
- generated only once with
- `pwl_calibrators_layers.calculate_quantiles_for_keypoints` in a separate
- invocation of your program. If you don't want to use quantiles, you can
- set `keypoints_initializer` instead.
-* `keypoints_initializers_fn`: if you know the distribution of your
- input features you can provide that directly instead of `quantiles_dir`.
- See `pwl_calibrators_layers.uniform_keypoints_for_signal`. It must be
- as a closure that when called will return a pair of tensors with
- keypoints input and output initializes. Alternatively can be given as
- a dict mapping feature name to keypoints_initializers_fn, so one
- can have one initialization per feature. It uses a closure instead of
- the tensors themselves because the graph has to be created at the time
- the model is being build, which happens at a later time.
-* `optimizer`: string, `Optimizer` object, or callable that defines the
- optimizer to use for training -- if a callable, it will be called with
- learning_rate=hparams.learning_rate.
-* `config`: RunConfig object to configure the runtime settings. Typically set
- to learn_runner.EstimatorConfig().
-* `hparams`: an instance of tfl_hparams.CalibrationHParams. If set to
- None default parameters are used.
-
-
-#### Returns:
-
-A `CalibratedLatticeClassifier` estimator.
-
-
-#### Raises:
-
-* `ValueError`: invalid parameters.
-* `KeyError`: type of feature not supported.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/calibrated_lattice_regressor.md b/g3doc/api_docs/python/tensorflow_lattice/calibrated_lattice_regressor.md
deleted file mode 100644
index df56ee7..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/calibrated_lattice_regressor.md
+++ /dev/null
@@ -1,108 +0,0 @@
-
-
-
-
-# tensorflow_lattice.calibrated_lattice_regressor
-
-``` python
-calibrated_lattice_regressor(
- feature_columns=None,
- model_dir=None,
- quantiles_dir=None,
- keypoints_initializers_fn=None,
- optimizer=None,
- config=None,
- hparams=None
-)
-```
-
-Calibrated lattice estimator (model) for regression.
-
-This model uses a piecewise lattice calibration function on each of the
-inputs (parametrized) and then combine (sum up) the results. Optionally
-calibration can be made monotonic.
-
-It usually requires a preprocessing step on the data, to calculate the
-quantiles of each used feature. This can be done locally or in one worker
-only before training, in a separate invocation of your program (or directly)
-in . Typically this can be save (`save_dir` parameter) to the same
-directory where the data is.
-
-Hyper-parameters are given in the form of the object
-tfl_hparams.CalibrationHParams. It takes in per-feature calibration
-parameters.
-
-Internally values will be converted to tf.float32.
-
-
-
-Example:
-
-```python
-def input_fn_train: ...
-def input_fn_eval: ...
-
-my_feature_columns=[...]
-
-# Have a separate program flag to generate the quantiles. Need to be run
-# only once.
-if FLAGS.create_quantiles:
- pwl_calibrators_layers.calculate_quantiles_for_keypoints(
- input_fn=input_fn_train,
- feature_columns=my_feature_columns,
- save_dir=FLAGS.data_dir,
- num_quantiles=1000,
- override=True)
- return # Exit program.
-
-estimator = calibrated_lattice.calibrated_lattice_regressor(
- feature_columns=feature_columns)
-estimator.train(input_fn=input_fn_train)
-estimator.evaluate(input_fn=input_fn_eval)
-estimator.predict(input_fn=input_fn_predict)
-```
-
-#### Args:
-
-* `feature_columns`: Optional, if not set the model will use all features
- returned by input_fn. An iteratable containing all the feature
- columns used by the model. All items in the set should be instances of
- classes derived from `FeatureColumn`. If not given, the model will
- use as features the tensors returned by input_fn.
- Supported types: RealValuedColumn.
-* `model_dir`: Directory to save model parameters, graph and etc. This can
- also be used to load checkpoints from the directory into a estimator to
- continue training a previously saved model.
-* `quantiles_dir`: location where quantiles for the data was saved. Typically
- the same directory as the training data. These quantiles can be
- generated only once with
- `pwl_calibrators_layers.calculate_quantiles_for_keypoints` in a separate
- invocation of your program. If you don't want to use quantiles, you can
- set `keypoints_initializer` instead.
-* `keypoints_initializers_fn`: if you know the distribution of your
- input features you can provide that directly instead of `quantiles_dir`.
- See `pwl_calibrators_layers.uniform_keypoints_for_signal`. It must be
- as a closure that when called will return a pair of tensors with
- keypoints input and output initializes. Alternatively can be given as
- a dict mapping feature name to keypoints_initializers_fn, so one
- can have one initialization per feature. It uses a closure instead of
- the tensors themselves because the graph has to be created at the time
- the model is being build, which happens at a later time.
-* `optimizer`: string, `Optimizer` object, or callable that defines the
- optimizer to use for training -- if a callable, it will be called with
- learning_rate=hparams.learning_rate.
-* `config`: RunConfig object to configure the runtime settings. Typically set
- to learn_runner.EstimatorConfig().
-* `hparams`: an instance of tfl_hparams.CalibrationHParams. If set to
- None default parameters are used.
-
-
-#### Returns:
-
-A `CalibratedLatticeRegressor` estimator.
-
-
-#### Raises:
-
-* `ValueError`: invalid parameters.
-* `KeyError`: type of feature not supported.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/calibrated_linear_classifier.md b/g3doc/api_docs/python/tensorflow_lattice/calibrated_linear_classifier.md
deleted file mode 100644
index 49c3f66..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/calibrated_linear_classifier.md
+++ /dev/null
@@ -1,111 +0,0 @@
-
-
-
-
-# tensorflow_lattice.calibrated_linear_classifier
-
-``` python
-calibrated_linear_classifier(
- feature_columns=None,
- model_dir=None,
- quantiles_dir=None,
- keypoints_initializers_fn=None,
- optimizer=None,
- config=None,
- hparams=None
-)
-```
-
-Calibrated linear classifier binary model.
-
-
-
-This model uses a piecewise linear calibration function on each of the
-real (as opposed to binary) inputs (parametrized) and then combines (sum up)
-the results. Optionally calibration can be made monotonic.
-
-It usually requires a preprocessing step on the data, to calculate the
-quantiles of each used feature. This can be done locally or in one worker
-only before training, in a separate invocation of your program (or directly).
-Typically this can be save (`save_dir` parameter) to the same
-directory where the data is.
-
-Hyper-parameters are given in the form of the object
-tfl_hparams.CalibrationHParams. It takes in per-feature calibration
-parameters.
-
-Internally values will be converted to tf.float32.
-
-Example:
-
-```python
-def input_fn_train: ...
-def input_fn_eval: ...
-
-my_feature_columns=[...]
-
-# Have a separate program flag to generate the quantiles. Need to be run
-# only once.
-if FLAGS.create_quantiles:
- pwl_calibrators_layers.calculate_quantiles_for_keypoints(
- input_fn=input_fn_train,
- feature_columns=my_feature_columns,
- save_dir=FLAGS.data_dir,
- num_quantiles=1000,
- override=True)
- return # Exit program.
-
-estimator = calibrated_linear.CalibratedLinearClassifier(
- feature_columns=feature_columns)
-estimator.train(input_fn=input_fn_train)
-estimator.evaluate(input_fn=input_fn_eval)
-estimator.predict(input_fn=input_fn_predict)
-```
-
-#### Args:
-
-* `feature_columns`: Optional, an iteratable containing all the feature
- columns used by the model. All items in the set should be instances of
- classes derived from `FeatureColumn`. If not given, the model will
- use as features the tensors returned by input_fn.
- Supported types of columns: RealValuedColumn.
-* `model_dir`: Directory to save model parameters, graph and etc. This can
- also be used to load checkpoints from the directory into a estimator to
- continue training a previously saved model.
-* `quantiles_dir`: location where quantiles for the data was saved. Typically
- the same directory as the training data. These quantiles can be
- generated only once with
- `pwl_calibration_layers.calculate_quantiles_for_keypoints` in a separate
- invocation of your program. If you don't want to use quantiles, you can
- set `keypoints_initializer` instead.
-* `keypoints_initializers_fn`: if you know the distribution of your
- input features you can provide that directly instead of `quantiles_dir`.
- See `pwl_calibrators_layers.uniform_keypoints_for_signal`. It must be
- a closure that returns a pair of tensors with keypoints inputs and
- outputs to use for initialization (must match `num_keypoints` configured
- in `hparams`). Alternatively the closure can return a dict mapping
- feature name to pairs for initialization per feature. If `quantiles_dir`
- and `keypoints_initializers_fn` are set, the later takes precendence,
- and the features for which `keypoints_initializers` are not defined
- fallback to using the quantiles found in `quantiles_dir`. It uses a
- closure instead of the tensors themselves because the graph has to be
- created at the time the model is being build, which happens at a later
- time.
-* `optimizer`: string, `Optimizer` object, or callable that defines the
- optimizer to use for training -- if a callable, it will be called with
- learning_rate=hparams.learning_rate.
-* `config`: RunConfig object to configure the runtime settings. Typically set
- to learn_runner.EstimatorConfig().
-* `hparams`: an instance of tfl_hparams.CalibrationHParams. If set to
- None default parameters are used.
-
-
-#### Returns:
-
-A `CalibratedLinearClassifier` estimator.
-
-
-#### Raises:
-
-* `ValueError`: invalid parameters.
-* `KeyError`: type of feature not supported.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/calibrated_linear_regressor.md b/g3doc/api_docs/python/tensorflow_lattice/calibrated_linear_regressor.md
deleted file mode 100644
index be5a622..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/calibrated_linear_regressor.md
+++ /dev/null
@@ -1,112 +0,0 @@
-
-
-
-
-# tensorflow_lattice.calibrated_linear_regressor
-
-``` python
-calibrated_linear_regressor(
- feature_columns=None,
- model_dir=None,
- quantiles_dir=None,
- keypoints_initializers_fn=None,
- optimizer=None,
- config=None,
- hparams=None
-)
-```
-
-Calibrated linear estimator (model) for regression.
-
-This model uses a piecewise linear calibration function on each of the
-inputs (parametrized) and then combine (sum up) the results. Optionally
-calibration can be made monotonic.
-
-It usually requires a preprocessing step on the data, to calculate the
-quantiles of each used feature. This can be done locally or in one worker
-only before training, in a separate invocation of your program (or directly)
-in . Typically this can be save (`save_dir` parameter) to the same
-directory where the data is.
-
-Hyper-parameters are given in the form of the object
-tfl_hparams.CalibrationHParams. It takes in per-feature calibration
-parameters.
-
-Internally values will be converted to tf.float32.
-
-
-
-Example:
-
-```python
-def input_fn_train: ...
-def input_fn_eval: ...
-
-my_feature_columns=[...]
-
-# Have a separate program flag to generate the quantiles. Need to be run
-# only once.
-if FLAGS.create_quantiles:
- pwl_calibrators_layers.calculate_quantiles_for_keypoints(
- input_fn=input_fn_train,
- feature_columns=my_feature_columns,
- save_dir=FLAGS.data_dir,
- num_quantiles=1000,
- override=True)
- return # Exit program.
-
-estimator = calibrated_linear.calibrated_linear_regressor(
- feature_columns=feature_columns)
-estimator.train(input_fn=input_fn_train)
-estimator.evaluate(input_fn=input_fn_eval)
-estimator.predict(input_fn=input_fn_predict)
-```
-
-#### Args:
-
-* `feature_columns`: Optional, if not set the model will use all features
- returned by input_fn. An iteratable containing all the feature
- columns used by the model. All items in the set should be instances of
- classes derived from `FeatureColumn`. If not given, the model will
- use as features the tensors returned by input_fn.
- Supported types: RealValuedColumn.
-* `model_dir`: Directory to save model parameters, graph and etc. This can
- also be used to load checkpoints from the directory into a estimator to
- continue training a previously saved model.
-* `quantiles_dir`: location where quantiles for the data was saved. Typically
- the same directory as the training data. These quantiles can be
- generated only once with
- `pwl_calibration_layers.calculate_quantiles_for_keypoints` in a separate
- invocation of your program. If you don't want to use quantiles, you can
- set `keypoints_initializer` instead.
-* `keypoints_initializers_fn`: if you know the distribution of your
- input features you can provide that directly instead of `quantiles_dir`.
- See `pwl_calibrators_layers.uniform_keypoints_for_signal`. It must be
- a closure that returns a pair of tensors with keypoints inputs and
- outputs to use for initialization (must match `num_keypoints` configured
- in `hparams`). Alternatively the closure can return a dict mapping
- feature name to pairs for initialization per feature. If `quantiles_dir`
- and `keypoints_initializers_fn` are set, the later takes precendence,
- and the features for which `keypoints_initializers` are not defined
- fallback to using the quantiles found in `quantiles_dir`. It uses a
- closure instead of the tensors themselves because the graph has to be
- created at the time the model is being build, which happens at a later
- time.
-* `optimizer`: string, `Optimizer` object, or callable that defines the
- optimizer to use for training -- if a callable, it will be called with
- learning_rate=hparams.learning_rate.
-* `config`: RunConfig object to configure the runtime settings. Typically set
- to learn_runner.EstimatorConfig().
-* `hparams`: an instance of tfl_hparams.CalibrationHParams. If set to
- None default parameters are used.
-
-
-#### Returns:
-
-A `CalibratedLinearRegressor` estimator.
-
-
-#### Raises:
-
-* `ValueError`: invalid parameters.
-* `KeyError`: type of feature not supported.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/calibrated_rtl_classifier.md b/g3doc/api_docs/python/tensorflow_lattice/calibrated_rtl_classifier.md
deleted file mode 100644
index 1e3927f..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/calibrated_rtl_classifier.md
+++ /dev/null
@@ -1,114 +0,0 @@
-
-
-
-
-# tensorflow_lattice.calibrated_rtl_classifier
-
-``` python
-calibrated_rtl_classifier(
- feature_columns=None,
- model_dir=None,
- quantiles_dir=None,
- keypoints_initializers_fn=None,
- optimizer=None,
- config=None,
- hparams=None
-)
-```
-
-Calibrated rtl binary classifier model.
-
-
-
-This model uses a piecewise lattice calibration function on each of the
-inputs (parametrized) and then feeds them to ensemble of random lattices.
-num_lattices and lattice_rank (number of inputs to each lattice) must be
-specified in the hyperparameter. Optionally calibration can be made monotonic.
-
-It usually requires a preprocessing step on the data, to calculate the
-quantiles of each used feature. This can be done locally or in one worker
-only before training, in a separate invocation of your program (or directly).
-Typically this can be saved (`save_dir` parameter) to the same
-directory where the data is.
-
-Hyper-parameters are given in the form of the object
-tfl_hparams.CalibrationRtlHParams. lattice_rank and num_lattices must
-be specified; there would be no default value for this. It also takes in
-per-feature parameters.
-
-Internally values will be converted to tf.float32.
-
-Example:
-
-```python
-def input_fn_train: ...
-def input_fn_eval: ...
-
-my_feature_columns=[...]
-
-# Have a separate program flag to generate the quantiles. Need to be run
-# only once.
-if FLAGS.create_quantiles:
- pwl_calibrators_layers.calculate_quantiles_for_keypoints(
- input_fn=input_fn_train,
- feature_columns=my_feature_columns,
- save_dir=FLAGS.data_dir,
- num_quantiles=1000,
- override=True)
- return # Exit program.
-
-hparams = hparams.CalibratedRtlHparams(num_lattices=10, lattice_rank=2)
-estimator = calibrated_rtl.calibrated_rtl_classifier(
- feature_columns=feature_columns, hparams=hparams)
-estimator.train(input_fn=input_fn_train)
-estimator.evaluate(input_fn=input_fn_eval)
-estimator.predict(input_fn=input_fn_predict)
-```
-
-#### Args:
-
-* `feature_columns`: Optional, an iteratable containing all the feature
- columns used by the model. All items in the set should be instances of
- classes derived from `FeatureColumn`. If not given, the model will
- use as features the tensors returned by input_fn.
- Supported types of columns: RealValuedColumn.
-* `model_dir`: Directory to save model parameters, graphs and etc. This can
- also be used to load checkpoints from the directory into an estimator to
- continue training a previously saved model.
-* `quantiles_dir`: location where quantiles for the data was saved. Typically
- the same directory as the training data. These quantiles can be
- generated only once with
- `pwl_calibration_layers.calculate_quantiles_for_keypoints` in a separate
- invocation of your program. If you don't want to use quantiles, you can
- set `keypoints_initializer` instead.
-* `keypoints_initializers_fn`: if you know the distribution of your
- input features you can provide that directly instead of `quantiles_dir`.
- See `pwl_calibrators_layers.uniform_keypoints_for_signal`. It must be
- a closure that returns a pair of tensors with keypoints inputs and
- outputs to use for initialization (must match `num_keypoints` configured
- in `hparams`). Alternatively the closure can return a dict mapping
- feature name to pairs for initialization per feature. If `quantiles_dir`
- and `keypoints_initializers_fn` are set, the later takes precendence,
- and the features for which `keypoints_initializers` are not defined
- fallback to using the quantiles found in `quantiles_dir`. It uses a
- closure instead of the tensors themselves because the graph has to be
- created at the time the model is being build, which happens at a later
- time.
-* `optimizer`: string, `Optimizer` object, or callable that defines the
- optimizer to use for training -- if a callable, it will be called with
- learning_rate=hparams.learning_rate.
-* `config`: RunConfig object to configure the runtime settings. Typically set
- to learn_runner.EstimatorConfig().
-* `hparams`: an instance of tfl_hparams.CalibrationRtlHParams. If set to
- None default parameters are used.
-
-
-#### Returns:
-
-A `calibrated_rtl_classifier` estimator.
-
-
-#### Raises:
-
-* `ValueError`: invalid parameters.
-* `KeyError`: type of feature not supported.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/calibrated_rtl_regressor.md b/g3doc/api_docs/python/tensorflow_lattice/calibrated_rtl_regressor.md
deleted file mode 100644
index 9365747..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/calibrated_rtl_regressor.md
+++ /dev/null
@@ -1,112 +0,0 @@
-
-
-
-
-# tensorflow_lattice.calibrated_rtl_regressor
-
-``` python
-calibrated_rtl_regressor(
- feature_columns=None,
- model_dir=None,
- quantiles_dir=None,
- keypoints_initializers_fn=None,
- optimizer=None,
- config=None,
- hparams=None
-)
-```
-
-Calibrated rtl regressor model.
-
-This model uses a piecewise lattice calibration function on each of the
-inputs (parametrized) and then feeds them to ensemble of random lattices.
-num_lattices and lattice_rank (number of inputs to each lattice) must be
-specified in the hyperparameter. Optionally calibration can be made monotonic.
-
-It usually requires a preprocessing step on the data, to calculate the
-quantiles of each used feature. This can be done locally or in one worker
-only before training, in a separate invocation of your program (or directly).
-Typically this can be saved (`save_dir` parameter) to the same
-directory where the data is.
-
-Hyper-parameters are given in the form of the object
-tfl_hparams.CalibrationRtlHParams. lattice_rank and num_lattices must
-be specified; there would be no default value for this. It also takes in
-per-feature parameters.
-
-Internally values will be converted to tf.float32.
-
-Example:
-
-```python
-def input_fn_train: ...
-def input_fn_eval: ...
-
-my_feature_columns=[...]
-
-# Have a separate program flag to generate the quantiles. Need to be run
-# only once.
-if FLAGS.create_quantiles:
- pwl_calibrators_layers.calculate_quantiles_for_keypoints(
- input_fn=input_fn_train,
- feature_columns=my_feature_columns,
- save_dir=FLAGS.data_dir,
- num_quantiles=1000,
- override=True)
- return # Exit program.
-
-hparams = hparams.CalibratedRtlHparams(num_lattices=10, lattice_rank=2)
-estimator = calibrated_rtl.calibrated_rtl_classifier(
- feature_columns=feature_columns, hparams=hparams)
-estimator.train(input_fn=input_fn_train)
-estimator.evaluate(input_fn=input_fn_eval)
-estimator.predict(input_fn=input_fn_test)
-```
-
-#### Args:
-
-* `feature_columns`: Optional, an iteratable containing all the feature
- columns used by the model. All items in the set should be instances of
- classes derived from `FeatureColumn`. If not given, the model will
- use as features the tensors returned by input_fn.
- Supported types of columns: RealValuedColumn.
-* `model_dir`: Directory to save model parameters, graphs and etc. This can
- also be used to load checkpoints from the directory into an estimator to
- continue training a previously saved model.
-* `quantiles_dir`: location where quantiles for the data was saved. Typically
- the same directory as the training data. These quantiles can be
- generated only once with
- `pwl_calibration_layers.calculate_quantiles_for_keypoints` in a separate
- invocation of your program. If you don't want to use quantiles, you can
- set `keypoints_initializer` instead.
-* `keypoints_initializers_fn`: if you know the distribution of your
- input features you can provide that directly instead of `quantiles_dir`.
- See `pwl_calibrators_layers.uniform_keypoints_for_signal`. It must be
- a closure that returns a pair of tensors with keypoints inputs and
- outputs to use for initialization (must match `num_keypoints` configured
- in `hparams`). Alternatively the closure can return a dict mapping
- feature name to pairs for initialization per feature. If `quantiles_dir`
- and `keypoints_initializers_fn` are set, the later takes precendence,
- and the features for which `keypoints_initializers` are not defined
- fallback to using the quantiles found in `quantiles_dir`. It uses a
- closure instead of the tensors themselves because the graph has to be
- created at the time the model is being build, which happens at a later
- time.
-* `optimizer`: string, `Optimizer` object, or callable that defines the
- optimizer to use for training -- if a callable, it will be called with
- learning_rate=hparams.learning_rate.
-* `config`: RunConfig object to configure the runtime settings. Typically set
- to learn_runner.EstimatorConfig().
-* `hparams`: an instance of tfl_hparams.CalibrationRtlHParams. If set to
- None default parameters are used.
-
-
-#### Returns:
-
-A `calibrated_rtl_regressor` estimator.
-
-
-#### Raises:
-
-* `ValueError`: invalid parameters.
-* `KeyError`: type of feature not supported.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/calibration_layer.md b/g3doc/api_docs/python/tensorflow_lattice/calibration_layer.md
deleted file mode 100644
index b5b65c8..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/calibration_layer.md
+++ /dev/null
@@ -1,96 +0,0 @@
-
-
-
-
-# tensorflow_lattice.calibration_layer
-
-``` python
-calibration_layer(
- uncalibrated_tensor,
- num_keypoints,
- keypoints_initializers=None,
- keypoints_initializer_fns=None,
- bound=False,
- monotonic=None,
- missing_input_values=None,
- missing_output_values=None,
- l1_reg=None,
- l2_reg=None,
- l1_laplacian_reg=None,
- l2_laplacian_reg=None,
- name=None
-)
-```
-
-Creates a calibration layer for uncalibrated values.
-
-Returns a calibrated tensor of the same shape as the uncalibrated continuous
-signals passed in, and a list of projection ops, that must be applied at
-each step (or every so many steps) to project the model to a feasible space:
-used for bounding the outputs or for imposing monotonicity -- the list will be
-empty if bound and monotonic are not set.
-
-#### Args:
-
-* `uncalibrated_tensor`: Tensor of shape [batch_size, ...] with uncalibrated
- values.
-* `num_keypoints`: Number of keypoints to use. Either a scalar value that
- will be used for every uncalibrated signal, or a list of n values,
- per uncalibrated signal -- uncalibrated is first flattened (
- see tf.contrib.layers.flatten) to [batch_size, n], and there should
- be one value in the list per n. If a value of the list is 0 or None
- the correspondent signal won't be calibrated.
-* `keypoints_initializers`: For evaluation or inference (or when resuming
- training from a checkpoint) the values will be loaded from disk, so they
- don't need to be given (leave it as None).
- Otherwise provide either a tuple of two tensors of shape [num_keypoints],
- or a list of n pairs of tensors, each of shape [num_keypoints]. In this
- list there should be one pair per uncalibrated signal, just like
- num_keypoints above. Notice that num_keypoints can be different per
- signal.
-* `keypoints_initializer_fns`: Like keypoints_initializers but using lambda
- initializers. They should be compatible with tf.get_variable. If this is
- set, then keypoints_initializers must be None.
-* `bound`: boolean whether output of calibration must be bound. Alternatively
- a list of n booleans, one per uncalibrated value, like num_keypoints
- above.
-* `monotonic`: whether calibration is monotonic: None or 0 means no
- monotonicity. Positive or negative values mean increasing or decreasing
- monotonicity respectively. Alternatively a list of n monotonic values,
- one per uncalibrated value, like num_keypoints above.
-* `missing_input_values`: If set, and if the input has this value it is assumed
- to be missing and the output will either be calibrated to some value
- between `[calibration_output_min, calibration_output_max]` or set to a
- fixed value set by missing_output_value. Limitation: it only works for
- scalars. Either one value for all inputs, or a list with one value per
- uncalibrated value.
-* `missing_output_values`: Requires missing_input_value also to be set. If set
- if will convert missing input to this value. Either one value for all
- outputs, or a list with one value per uncalibrated value.
-* `l1_reg`: (list of floats or float) l1 regularization amount.
- If float, then same value is applied to all dimensions.
-* `l2_reg`: (list of floats or float) l2 regularization amount.
- If float, then same value is applied to all dimensions.
-* `l1_laplacian_reg`: (list of floats or float) l1 laplacian
- regularization amount. If float, then same value is applied to all
- dimensions.
-* `l2_laplacian_reg`: (list of floats or float) l2 laplacian
- regularization amount. If float, then same value is applied to all
- dimensions.
-* `name`: Name scope for operations.
-
-
-#### Returns:
-
-A tuple of:
-* calibrated tensor of shape [batch_size, ...], the same shape as
- uncalibrated.
-* list of projection ops, that must be applied at each step (or every so
- many steps) to project the model to a feasible space: used for bounding
- the outputs or for imposing monotonicity. Empty if none are requested.
-* None or tensor with regularization loss.
-
-
-#### Raises:
-
-* `ValueError`: If dimensions don't match.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/calibrator_regularization.md b/g3doc/api_docs/python/tensorflow_lattice/calibrator_regularization.md
deleted file mode 100644
index 67f64bd..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/calibrator_regularization.md
+++ /dev/null
@@ -1,39 +0,0 @@
-
-
-
-
-# tensorflow_lattice.calibrator_regularization
-
-``` python
-calibrator_regularization(
- output_keypoints,
- l1_reg=None,
- l2_reg=None,
- l1_laplacian_reg=None,
- l2_laplacian_reg=None,
- name='calibrator_regularization'
-)
-```
-
-Returns a calibrator regularization op.
-
-#### Args:
-
-output_keypoints: (Rank-1 tensor with shape [num_keypoints]) 1d calibrator's
- output keypoints tensor.
-l1_reg: (float) l1 regularization amount.
-l2_reg: (float) l2 regularization amount.
-l1_laplacian_reg: (float) l1 Laplacian regularization amount.
-l2_laplacian_reg: (float) l2 Laplacian regularization amount.
-name: name scope of calibrator regularization.
-
-
-#### Returns:
-
-Rank-0 tensor (scalar) that contains calibrator regularization.
-
-
-#### Raises:
-
-* `ValueError`: * If output_keypoints is not rank-1 tensor.
- * If the shape of output_keypoints is unknown.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/ensemble_lattices_layer.md b/g3doc/api_docs/python/tensorflow_lattice/ensemble_lattices_layer.md
deleted file mode 100644
index 836943f..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/ensemble_lattices_layer.md
+++ /dev/null
@@ -1,69 +0,0 @@
-
-
-
-
-# tensorflow_lattice.ensemble_lattices_layer
-
-``` python
-ensemble_lattices_layer(
- input_tensor,
- lattice_sizes,
- structure_indices,
- is_monotone=None,
- output_dim=1,
- interpolation_type='hypercube',
- lattice_initializers=None,
- l1_reg=None,
- l2_reg=None,
- l1_torsion_reg=None,
- l2_torsion_reg=None,
- l1_laplacian_reg=None,
- l2_laplacian_reg=None
-)
-```
-
-Creates a ensemble of lattices layer.
-
-Returns a list of output of lattices, lattice parameters, and projection ops.
-
-#### Args:
-
-* `input_tensor`: [batch_size, input_dim] tensor.
-* `lattice_sizes`: A list of lattice sizes of each dimension.
-* `structure_indices`: A list of list of ints. structure_indices[k] is a list
- of indices that belongs to kth lattices.
-* `is_monotone`: A list of input_dim booleans, boolean or None. If None or
- False, lattice will not have monotonicity constraints. If
- is_monotone[k] == True, then the lattice output has the non-decreasing
- monotonicity with respect to input_tensor[?, k] (the kth coordinate). If
- True, all the input coordinate will have the non-decreasing monotonicity.
-* `output_dim`: Number of outputs.
-* `interpolation_type`: 'hypercube' or 'simplex'.
-* `lattice_initializers`: (Optional) A list of initializer for each lattice
- parameter vectors. lattice_initializer[k] is a 2D tensor
- [output_dim, parameter_dim[k]], where parameter_dim[k] is the number of
- parameter in the kth lattice. If None, lattice_param_as_linear initializer
- will be used with
- linear_weights=[1 if monotone else 0 for monotone in is_monotone].
-* `l1_reg`: (float) l1 regularization amount.
-* `l2_reg`: (float) l2 regularization amount.
-* `l1_torsion_reg`: (float) l1 torsion regularization amount.
-* `l2_torsion_reg`: (float) l2 torsion regularization amount.
-* `l1_laplacian_reg`: (list of floats or float) list of L1 Laplacian
- regularization amount per each dimension. If a single float value is
- provided, then all diemnsion will get the same value.
-* `l2_laplacian_reg`: (list of floats or float) list of L2 Laplacian
- regularization amount per each dimension. If a single float value is
- provided, then all diemnsion will get the same value.
-
-
-#### Returns:
-
-A tuple of:
-* a list of output tensors, [batch_size, output_dim], with length
- len(structure_indices), i.e., one for each lattice.
-* a list of parameter tensors shape [output_dim, parameter_dim]
-* None or projection ops, that must be applied at each
- step (or every so many steps) to project the model to a feasible space:
- used for bounding the outputs or for imposing monotonicity.
-* None or a regularization loss, if regularization is configured.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/input_calibration_layer.md b/g3doc/api_docs/python/tensorflow_lattice/input_calibration_layer.md
deleted file mode 100644
index 633d3bf..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/input_calibration_layer.md
+++ /dev/null
@@ -1,102 +0,0 @@
-
-
-
-
-# tensorflow_lattice.input_calibration_layer
-
-``` python
-input_calibration_layer(
- columns_to_tensors,
- num_keypoints,
- feature_columns=None,
- keypoints_initializers=None,
- keypoints_initializer_fns=None,
- bound=False,
- monotonic=None,
- missing_input_values=None,
- missing_output_values=None,
- l1_reg=None,
- l2_reg=None,
- l1_laplacian_reg=None,
- l2_laplacian_reg=None,
- dtype=dtypes.float32
-)
-```
-
-Creates a calibration layer for the given input and feature_columns.
-
-Returns a tensor with the calibrated values of the given features, a list
-of the names of the features in the order they feature in the returned, and
-a list of projection ops, that must be applied at each step (or every so many
-steps) to project the model to a feasible space: used for bounding the outputs
-or for imposing monotonic -- the list will be empty if bound and
-monotonic are not set.
-
-#### Args:
-
-* `columns_to_tensors`: A mapping from feature name to tensors. 'string' key
- means a base feature (not-transformed). If feature_columns is not set
- these are the features calibrated. Otherwise the transformed
- feature_columns are the ones calibrated.
-* `num_keypoints`: Number of keypoints to use. Either a single int, or a dict
- mapping feature names to num_keypoints. If a value of the dict is 0 or
- None the correspondent feature won't be calibrated.
-* `feature_columns`: Optional. If set to a set of FeatureColumns, these will
- be the features used and calibrated.
-* `keypoints_initializers`: For evaluation or inference (or when resuming
- training from a checkpoint) the values will be loaded from disk, so they
- don't need to be given (leave it as None).
- Either a tuple of two tensors of shape [num_keypoints], or a dict mapping
- feature names to pair of tensors of shape [num_keypoints[feature_name]].
- See load_keypoints_from_quantiles or uniform_keypoints_for_signal on how
- to generate these (module keypoints_initialization).
-* `keypoints_initializer_fns`: Like keypoints_initializers but using lambda
- initializers. They should be compatible with tf.get_variable. If this is
- set, then keypoints_initializers must be None.
-* `bound`: boolean whether output of calibration must be bound. Alternatively
- a dict mapping feature name to boundness.
-* `monotonic`: whether calibration has to be kept monotonic: None or 0 means
- no monotonic. Positive or negative values mean increasing or decreasing
- monotonic respectively. Alternatively a dict mapping feature name
- to monotonic.
-* `missing_input_values`: If set, and if the input has this value it is assumed
- to be missing and the output will either be calibrated to some value
- between `[calibration_output_min, calibration_output_max]` or set to a
- fixed value set by missing_output_value. Limitation: it only works for
- scalars. Either one value for all inputs, or a dict mapping feature name
- to missing_input_value for the respective feature.
-* `missing_output_values`: Requires missing_input_value also to be set. If set
- if will convert missing input to this value. Either one value for all
- inputs, or a dict mapping feature name to missing_input_value for the
- respective feature.
-* `l1_reg`: ({feature_name: float} dict or float) l1 regularization amount.
- If float, then same value is applied to all features.
-* `l2_reg`: ({feature_name: float} dict or float) l2 regularization amount.
- If float, then same value is applied to all features.
-* `l1_laplacian_reg`: ({feature_name: float} dict or float) l1 laplacian
- regularization amount. If float, then same value is applied to all
- features.
-* `l2_laplacian_reg`: ({feature_name: float} dict or float) l2 laplacian
- regularization amount. If float, then same value is applied to all
- features.
-* `dtype`: If any of the scalars are not given as tensors, they are converted
- to tensors with this dtype.
-
-
-#### Returns:
-
-A tuple of:
-* calibrated tensor of shape [batch_size, sum(features dimensions)].
-* list of the feature names in the order they feature in the calibrated
- tensor. A name may appear more than once if the feature is
- multi-dimension (for instance a multi-dimension embedding)
-* list of projection ops, that must be applied at each step (or every so
- many steps) to project the model to a feasible space: used for bounding
- the outputs or for imposing monotonicity. Empty if none are requested.
-* None or tensor with regularization loss.
-
-
-#### Raises:
-
-* `ValueError`: if dtypes are incompatible.
-
diff --git a/g3doc/api_docs/python/tensorflow_lattice/input_calibration_layer_from_hparams.md b/g3doc/api_docs/python/tensorflow_lattice/input_calibration_layer_from_hparams.md
deleted file mode 100644
index 770b2a1..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/input_calibration_layer_from_hparams.md
+++ /dev/null
@@ -1,76 +0,0 @@
-
-
-
-
-# tensorflow_lattice.input_calibration_layer_from_hparams
-
-``` python
-input_calibration_layer_from_hparams(
- columns_to_tensors,
- feature_columns,
- hparams,
- quantiles_dir=None,
- keypoints_initializers=None,
- name=None,
- dtype=dtypes.float32
-)
-```
-
-Creates a calibration layer for the input using hyper-parameters.
-
-Similar to `input_calibration_layer` but reads its parameters from a
-`CalibratedHParams` object.
-
-#### Args:
-
-* `columns_to_tensors`: A mapping from feature name to tensors. 'string' key
- means a base feature (not-transformed). If feature_columns is not set
- these are the features calibrated. Otherwise the transformed
- feature_columns are the ones calibrated.
-* `feature_columns`: An iterable containing all the feature columns used by the
- model. Optional, if not set the model will use all features given in
- columns_to_tensors. All items in the set should be instances of
- classes derived from `FeatureColumn`.
-* `hparams`: Hyper-parameters, need to inherit from `CalibratedHParams`.
- It is also changed to include all feature names found in
- `feature_columns`. See `CalibratedHParams` and `input_calibration_layer`
- for descriptions of how these hyper-parameters work.
-* `quantiles_dir`: location where quantiles for the data was saved. Typically
- the same directory as the training data. These quantiles can be
- generated with `pwl_calibration_layers.calculate_quantiles_for_keypoints`,
- maybe in a separate invocation of your program. Different models that
- share the same quantiles information -- so this needs to be generated only
- once when hyper-parameter tuning. If you don't want to use quantiles, you
- can set `keypoints_initializers` instead.
-* `keypoints_initializers`: if you know the distribution of your
- input features you can provide that directly instead of `quantiles_dir`.
- See `pwl_calibrators_layers.uniform_keypoints_for_signal`. It must be
- a pair of tensors with keypoints inputs and outputs to use for
- initialization (must match `num_keypoints` configured in `hparams`).
- Alternatively can be given as a dict mapping feature name to pairs,
- for initialization per feature. If `quantiles_dir` and
- `keypoints_initializer` are set, the later takes precendence, and the
- features for which `keypoints_initializers` are not defined fallback to
- using the quantiles found in `quantiles_dir`.
-* `name`: Name scope for layer.
-* `dtype`: If any of the scalars are not given as tensors, they are converted
- to tensors with this dtype.
-
-
-#### Returns:
-
-A tuple of:
-* calibrated tensor of shape [batch_size, sum(features dimensions)].
-* list of the feature names in the order they feature in the calibrated
- tensor. A name may appear more than once if the feature is
- multi-dimension (for instance a multi-dimension embedding)
-* list of projection ops, that must be applied at each step (or every so
- many steps) to project the model to a feasible space: used for bounding
- the outputs or for imposing monotonicity. Empty if none are requested.
-* None or tensor with regularization loss.
-
-
-#### Raises:
-
-* `ValueError`: if dtypes are incompatible.
-
diff --git a/g3doc/api_docs/python/tensorflow_lattice/lattice.md b/g3doc/api_docs/python/tensorflow_lattice/lattice.md
deleted file mode 100644
index 1ac115d..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/lattice.md
+++ /dev/null
@@ -1,36 +0,0 @@
-
-
-
-
-# tensorflow_lattice.lattice
-
-``` python
-lattice(
- input_tensor,
- parameter_tensor,
- lattice_sizes,
- interpolation_type='hypercube'
-)
-```
-
-Returns an interpolated look-up table (lattice) op.
-
-#### Args:
-
-* `input_tensor`: [batch_size, input_dim] tensor.
-* `parameter_tensor`: [output_dim, param_dim] tensor, where param_dim ==
- lattice_sizes[0] * ... * lattice_sizes[input_dim - 1].
-* `lattice_sizes`: A list of lattice sizes of each dimension.
-* `interpolation_type`: 'hypercube' or 'simplex'.
-
-
-#### Returns:
-
-* `output_tensor`: [batch_size, num_outputs] tensor that contains the output of
- hypercube lattice.
-
-
-#### Raises:
-
-* `ValueError`: If interpolation_type is not 'hypercube' nor 'simplex'.
-
diff --git a/g3doc/api_docs/python/tensorflow_lattice/lattice_layer.md b/g3doc/api_docs/python/tensorflow_lattice/lattice_layer.md
deleted file mode 100644
index d4824f4..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/lattice_layer.md
+++ /dev/null
@@ -1,69 +0,0 @@
-
-
-
-
-# tensorflow_lattice.lattice_layer
-
-``` python
-lattice_layer(
- input_tensor,
- lattice_sizes,
- is_monotone=None,
- output_dim=1,
- interpolation_type='hypercube',
- lattice_initializer=None,
- l1_reg=None,
- l2_reg=None,
- l1_torsion_reg=None,
- l2_torsion_reg=None,
- l1_laplacian_reg=None,
- l2_laplacian_reg=None
-)
-```
-
-Creates a lattice layer.
-
-Returns an output of lattice, lattice parameters, and projection ops.
-
-#### Args:
-
-* `input_tensor`: [batch_size, input_dim] tensor.
-* `lattice_sizes`: A list of lattice sizes of each dimension.
-* `is_monotone`: A list of input_dim booleans, boolean or None. If None or
- False, lattice will not have monotonicity constraints. If
- is_monotone[k] == True, then the lattice output has the non-decreasing
- monotonicity with respect to input_tensor[?, k] (the kth coordinate). If
- True, all the input coordinate will have the non-decreasing monotonicity.
-* `output_dim`: Number of outputs.
-* `interpolation_type`: 'hypercube' or 'simplex'.
-* `lattice_initializer`: (Optional) Initializer for lattice parameter vectors,
- a 2D tensor [output_dim, parameter_dim] (where parameter_dim ==
- lattice_sizes[0] * ... * lattice_sizes[input_dim - 1]). If None,
- lattice_param_as_linear initializer will be used with
- linear_weights=[1 if monotone else 0 for monotone in is_monotone].
-* `l1_reg`: (float) l1 regularization amount.
-* `l2_reg`: (float) l2 regularization amount.
-* `l1_torsion_reg`: (float) l1 torsion regularization amount.
-* `l2_torsion_reg`: (float) l2 torsion regularization amount.
-* `l1_laplacian_reg`: (list of floats or float) list of L1 Laplacian
- regularization amount per each dimension. If a single float value is
- provided, then all diemnsion will get the same value.
-* `l2_laplacian_reg`: (list of floats or float) list of L2 Laplacian
- regularization amount per each dimension. If a single float value is
- provided, then all diemnsion will get the same value.
-
-
-#### Returns:
-
-A tuple of:
-* output tensor of shape [batch_size, output_dim]
-* parameter tensor of shape [output_dim, parameter_dim]
-* None or projection ops, that must be applied at each
- step (or every so many steps) to project the model to a feasible space:
- used for bounding the outputs or for imposing monotonicity.
-* None or a regularization loss, if regularization is configured.
-
-
-#### Raises:
-
-* `ValueError`: for invalid parameters.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/lattice_regularization.md b/g3doc/api_docs/python/tensorflow_lattice/lattice_regularization.md
deleted file mode 100644
index b52ee03..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/lattice_regularization.md
+++ /dev/null
@@ -1,49 +0,0 @@
-
-
-
-
-# tensorflow_lattice.lattice_regularization
-
-``` python
-lattice_regularization(
- lattice_params,
- lattice_sizes,
- l1_reg=None,
- l2_reg=None,
- l1_torsion_reg=None,
- l2_torsion_reg=None,
- l1_laplacian_reg=None,
- l2_laplacian_reg=None,
- name='lattice_regularization'
-)
-```
-
-Returns a lattice regularization op.
-
-#### Args:
-
-lattice_params: (Rank-2 tensor with shape [output_dim, param_dim]) Lattice
- parameter tensor.
-lattice_sizes: (list of integers) lattice size of each dimension.
-l1_reg: (float) l1 regularization amount.
-l2_reg: (float) l2 regularization amount.
-l1_torsion_reg: (float) l1 torsion regularization amount.
-l2_torsion_reg: (float) l2 torsion regularization amount.
-l1_laplacian_reg: (list of floats or float) list of L1 Laplacian
- regularization amount per each dimension. If a single float value is
- provided, then all diemnsion will get the same value.
-l2_laplacian_reg: (list of floats or float) list of L2 Laplacian
- regularization amount per each dimension. If a single float value is
- provided, then all diemnsion will get the same value.
-name: name scope of lattice regularization.
-
-
-#### Returns:
-
-Rank-0 tensor (scalar) that contains lattice regularization.
-
-
-#### Raises:
-
-* `ValueError`: * lattice_param is not rank-2 tensor.
- * output_dim or param_dim is unknown.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/load_keypoints_from_quantiles.md b/g3doc/api_docs/python/tensorflow_lattice/load_keypoints_from_quantiles.md
deleted file mode 100644
index b6ace1f..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/load_keypoints_from_quantiles.md
+++ /dev/null
@@ -1,57 +0,0 @@
-
-
-
-
-# tensorflow_lattice.load_keypoints_from_quantiles
-
-``` python
-load_keypoints_from_quantiles(
- feature_names,
- save_dir,
- num_keypoints,
- output_min,
- output_max,
- dtype=dtypes.float32
-)
-```
-
-Retrieves keypoints initialization values for selected features.
-
-It expects that the quantiles have already been calculated and saved in the
-save_dir by the save_quantiles_for_keypoints function. It will raise
-an I/O error if not.
-
-#### Args:
-
-* `feature_names`: List of features names for which to get keypoints
- initialization values.
-* `save_dir`: Directory where the quantiles have been saved to. Same value used
- when save_quantiles_for_keypoints was called.
-* `num_keypoints`: Desired number of keypoints to use for calibration. This
- can either be a scalar to be used for all features, or a dict mapping
- feature name to num_keypoints. Fewer keypoints than requested can end
- up being used when for the given feature there are not enough different
- values. If num_keypoints for a feature is missing, None or 0, no
- initialization is generated.
-* `output_min`: Initial calibrated value associated with the first calibration
- keypoint. The keypoints outputs in between will be linearly interpolated.
- It can be given as a scalar, in which case value is used for all features,
- or a dict mapping feature name to output_min.
-* `output_max`: Like output_min, but the calibrated value associated to the
- last keypoint. Scalar or dict.
-* `dtype`: Type to be used for calibration.
-
-
-#### Returns:
-
-Dict of feature name to pair of constant tensors that can be used to
-initialize calibrators keypoints inputs and outputs.
-
-
-#### Raises:
-
-* `tf.errors.NotFoundError`: if quantiles file not found.
-
-
- values in the signal. This would probably be better handled as categorical,
- but still this should handle the case correctly.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/monotone_lattice.md b/g3doc/api_docs/python/tensorflow_lattice/monotone_lattice.md
deleted file mode 100644
index e666518..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/monotone_lattice.md
+++ /dev/null
@@ -1,58 +0,0 @@
-
-
-
-
-# tensorflow_lattice.monotone_lattice
-
-``` python
-monotone_lattice(
- lattice_params,
- is_monotone=[],
- lattice_sizes=[],
- tolerance=1e-07,
- max_iter=1000,
- name=None
-)
-```
-
-Returns a projected lattice parameters onto the monotonicity constraints.
-
-Monotonicity constraints are specified is_monotone. If is_monotone[k] == True,
-then the kth input has a non-decreasing monotonicity, otherwise there will be no
-constraints.
-
-This operator uses an iterative algorithm, Alternating Direction Method of
-Multipliers (ADMM) method, to find the projection, so tolerance and max_iter can
-be used to control the accuracy vs. the time spent trade-offs in the ADMM
-method.
-
-Inputs
- lattice_params: 2D tensor, `[number of outputs, number of parameters]`
-
-Params
- is_monotone: 1D bool tensor that contains whether the kth dimension should be
- monotonic.
- lattice_sizes: 1D int tensor that contains a lattice size per each dimension,
- [m_0, ..., m_{d - 1}].
- tolerance: The tolerance in ||true projection - projection|| in the ADMM
- method.
- max_iter: Maximum number of iterations in the ADMM method.
-
-Outputs
- projected_lattice_params: 2D tensor,
- `[number of outputs, number of parameters]`, that contains the projected
- parameters.
-
-#### Args:
-
-* `lattice_params`: A `Tensor`. Must be one of the following types: `float32`, `float64`.
-* `is_monotone`: An optional list of `bools`. Defaults to `[]`.
-* `lattice_sizes`: An optional list of `ints`. Defaults to `[]`.
-* `tolerance`: An optional `float`. Defaults to `1e-07`.
-* `max_iter`: An optional `int`. Defaults to `1000`.
-* `name`: A name for the operation (optional).
-
-
-#### Returns:
-
-A `Tensor`. Has the same type as `lattice_params`.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/monotonic_projection.md b/g3doc/api_docs/python/tensorflow_lattice/monotonic_projection.md
deleted file mode 100644
index 4d61848..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/monotonic_projection.md
+++ /dev/null
@@ -1,41 +0,0 @@
-
-
-
-
-# tensorflow_lattice.monotonic_projection
-
-``` python
-monotonic_projection(
- values,
- increasing,
- name=None
-)
-```
-
-Returns a not-strict monotonic projection of the vector.
-
-The returned vector is of the same size as the input and values (optionally)
-changed to make them monotonically, minimizing the sum of the square distance
-to the original values.
-
-This is part of the set of ops that support monotonicity in piecewise-linear
-calibration.
-
-Note that the gradient is undefined for this function.
-
- values: `Tensor` with values to be made monotonic.
- increasing: Defines if projection it to monotonic increasing values
- or to monotonic decreasing ones.
-
- monotonic: output `Tensor` with values made monotonic.
-
-#### Args:
-
-* `values`: A `Tensor`. Must be one of the following types: `float32`, `float64`.
-* `increasing`: A `Tensor` of type `bool`.
-* `name`: A name for the operation (optional).
-
-
-#### Returns:
-
-A `Tensor`. Has the same type as `values`.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/pwl_indexing_calibrator.md b/g3doc/api_docs/python/tensorflow_lattice/pwl_indexing_calibrator.md
deleted file mode 100644
index a6534c0..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/pwl_indexing_calibrator.md
+++ /dev/null
@@ -1,42 +0,0 @@
-
-
-
-
-# tensorflow_lattice.pwl_indexing_calibrator
-
-``` python
-pwl_indexing_calibrator(
- input,
- kp_inputs,
- name=None
-)
-```
-
-Returns tensor representing interpolation weights in a piecewise linear
-
-function. If using a large number of keypoints, try PwlIndexingCalibratorSparse.
-
-Notice that in this version the keypoints inputs (given by kp_inputs) is kept
-fixed by forcing its gradient to be always 0. FutureWork: allow kp_inputs to
-also be optimized, by providing a gradient.
-
-Inputs
- input: uncalibrated weights, `[batch_size]`
- kp_input: keypoints' input weights, can be initialized with the
- pwl_calibrator_initialize_input_keypoints op. `[num_keypoints]`
-
-Outputs
- weights: Interpolation weights for a piecewise linear function. Its shape is
- `[batch_size, num_keypoints]`. The dot product of this and the keypoints
- output will give the calibrated value.
-
-#### Args:
-
-* `input`: A `Tensor`. Must be one of the following types: `float32`, `float64`.
-* `kp_inputs`: A `Tensor`. Must have the same type as `input`.
-* `name`: A name for the operation (optional).
-
-
-#### Returns:
-
-A `Tensor`. Has the same type as `input`.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/save_quantiles_for_keypoints.md b/g3doc/api_docs/python/tensorflow_lattice/save_quantiles_for_keypoints.md
deleted file mode 100644
index 338e1f7..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/save_quantiles_for_keypoints.md
+++ /dev/null
@@ -1,78 +0,0 @@
-
-
-
-
-# tensorflow_lattice.save_quantiles_for_keypoints
-
-``` python
-save_quantiles_for_keypoints(
- input_fn,
- save_dir,
- feature_columns=None,
- num_steps=1,
- override=True,
- num_quantiles=1000,
- dtype=dtypes.float32
-)
-```
-
-Calculates and saves quantiles for given features.
-
-These values can later be retrieved and used by keypoints_from_quantiles()
-below.
-
-Repeated values are discarded before the quantiles are calculated. That means
-that the quantiles of a very skewed distribution (for instance where 99%
-of the values are 0), will be different. But for the purpose of calibration
-this approach is more useful.
-
-Nothing is returned, the values are simply saved in the given location.
-
-This function can be called as a preprocessing step before actual training
-starts. Typically one will run this in a separate process locally, before
-starting training for instance.
-
-#### Args:
-
-* `input_fn`: Similar to input_fn provided to Estimators. Typically one
- doesn't need to go over the full data to get good quantiles. Typically
- some 100 random examples per quantile is good enough for the purpose of
- calibration. If you don't have too much data, just use everything.
- If input_fn returns a target (used in training) it is ignored.
-* `save_dir`: Where to save these quantiles. Since when optimizing
- hyper-parameters we train various models, we can share the quantiles
- information generated here. So this should be a directory that can be
- accessed by all training sessions. A subdirectory called "quantiles" will
- be created, and inside one file per feature is created: named after the
- feature name, and with the quantiles stored in JSON format.
-* `feature_columns`: If set, quantiles are generated for these feature columns.
- The file name used to save the quantiles uses a hash of the names of the
- feature_columns, so it can support different quantiles sets for different
- parts of the model if needed. If not set quantiles will be generated for
- all features returned by input_fn.
-* `num_steps`: number of steps to take over input_fn to gather enough data to
- create quantiles. Set to 0 or None to run until queue is exhausted,
- like if you used num_epochs in your input_fn.
-* `override`: if False it won't regenerate quantiles for files that are already
- there. This works as long as the features definition/distribution hasn't
- change from one run to another.
-* `num_quantiles`: This value should be larger than the maximum number of
- keypoints that will be considered for calibrating these features. If
- there are not enough quantiles for the keypoints, the system is robust and
- will simply interpolate the missing quantiles. Similarly if there are not
- enough examples to represent the quantiles, it will interpolate the
- quantiles from the examples given.
-* `dtype`: Deafult dtype to use, in particular for categorical values.
-
-Returns: Nothing, results are saved to disk.
-
-
-#### Raises:
-
-* `errors.OpError`: For I/O errors.
-
-FutureWork:
- * Use Munro-Paterson algorithm to calculate quantiles in a streaming
- fashion. See Squawd library.
- * Add support to weighted examples.
- * Handle cases where there are not enough different values in quantiles.
\ No newline at end of file
diff --git a/g3doc/api_docs/python/tensorflow_lattice/uniform_keypoints_for_signal.md b/g3doc/api_docs/python/tensorflow_lattice/uniform_keypoints_for_signal.md
deleted file mode 100644
index 1f6dfd8..0000000
--- a/g3doc/api_docs/python/tensorflow_lattice/uniform_keypoints_for_signal.md
+++ /dev/null
@@ -1,47 +0,0 @@
-
-
-
-
-# tensorflow_lattice.uniform_keypoints_for_signal
-
-``` python
-uniform_keypoints_for_signal(
- num_keypoints,
- input_min,
- input_max,
- output_min,
- output_max,
- dtype=dtypes.float32
-)
-```
-
-Returns a pair of initialization tensors for calibration keypoints.
-
-This is used when the input range to be calibrated is known.
-
-#### Args:
-
-* `num_keypoints`: number of keypoints to use for calibrating this signal.
-* `input_min`: Scalar with the minimum value that the uncalibrated input
- can take.
-* `input_max`: Scalar with the maximum value that the uncalibrated input
- can take.
-* `output_min`: Scalar with calibrated value associated with input_min.
- Typically the minimum expected calibrated value, but not necessarily.
- Specially if the calibration is decreasing.
-* `output_max`: Scalar with calibrated scalar value associated with
- input_max.
-* `dtype`: If any of the scalars are not given as tensors, they are converted
- to tensors with this dtype.
-
-
-#### Returns:
-
-Two tensors to be used as the keypoints_inputs and keypoints_outputs
-initialization, uniformly distributed over given ranges. Dtype is given
-by input_min, input_max, output_min, output_max.
-
-
-#### Raises:
-
-* `ValueError`: if underlying types (dtype) don't match.
\ No newline at end of file
diff --git a/g3doc/images/data_dist.png b/g3doc/images/data_dist.png
deleted file mode 100644
index 3e71189..0000000
Binary files a/g3doc/images/data_dist.png and /dev/null differ
diff --git a/g3doc/images/deep_lattice_networks.png b/g3doc/images/deep_lattice_networks.png
deleted file mode 100644
index d198f83..0000000
Binary files a/g3doc/images/deep_lattice_networks.png and /dev/null differ
diff --git a/g3doc/images/mono_1_of_4.png b/g3doc/images/mono_1_of_4.png
deleted file mode 100644
index e111a64..0000000
Binary files a/g3doc/images/mono_1_of_4.png and /dev/null differ
diff --git a/g3doc/images/mono_2_of_4.png b/g3doc/images/mono_2_of_4.png
deleted file mode 100644
index 3e78540..0000000
Binary files a/g3doc/images/mono_2_of_4.png and /dev/null differ
diff --git a/g3doc/images/mono_3_of_4.png b/g3doc/images/mono_3_of_4.png
deleted file mode 100644
index 7d17cdc..0000000
Binary files a/g3doc/images/mono_3_of_4.png and /dev/null differ
diff --git a/g3doc/images/mono_4_of_4.png b/g3doc/images/mono_4_of_4.png
deleted file mode 100644
index 675b636..0000000
Binary files a/g3doc/images/mono_4_of_4.png and /dev/null differ
diff --git a/g3doc/images/pwl_calibration_distance.png b/g3doc/images/pwl_calibration_distance.png
deleted file mode 100644
index 4bcc336..0000000
Binary files a/g3doc/images/pwl_calibration_distance.png and /dev/null differ
diff --git a/g3doc/images/pwl_calibration_price.png b/g3doc/images/pwl_calibration_price.png
deleted file mode 100644
index f1ac7a2..0000000
Binary files a/g3doc/images/pwl_calibration_price.png and /dev/null differ
diff --git a/g3doc/tutorial/images/2d_lattice.png b/g3doc/tutorial/images/2d_lattice.png
deleted file mode 100644
index 7d5c8c8..0000000
Binary files a/g3doc/tutorial/images/2d_lattice.png and /dev/null differ
diff --git a/g3doc/tutorial/images/data_dist.png b/g3doc/tutorial/images/data_dist.png
deleted file mode 100644
index 3e71189..0000000
Binary files a/g3doc/tutorial/images/data_dist.png and /dev/null differ
diff --git a/g3doc/tutorial/images/deep_lattice_networks.png b/g3doc/tutorial/images/deep_lattice_networks.png
deleted file mode 100644
index d198f83..0000000
Binary files a/g3doc/tutorial/images/deep_lattice_networks.png and /dev/null differ
diff --git a/g3doc/tutorial/images/mono_1_of_4.png b/g3doc/tutorial/images/mono_1_of_4.png
deleted file mode 100644
index e111a64..0000000
Binary files a/g3doc/tutorial/images/mono_1_of_4.png and /dev/null differ
diff --git a/g3doc/tutorial/images/mono_2_of_4.png b/g3doc/tutorial/images/mono_2_of_4.png
deleted file mode 100644
index 3e78540..0000000
Binary files a/g3doc/tutorial/images/mono_2_of_4.png and /dev/null differ
diff --git a/g3doc/tutorial/images/mono_3_of_4.png b/g3doc/tutorial/images/mono_3_of_4.png
deleted file mode 100644
index 7d17cdc..0000000
Binary files a/g3doc/tutorial/images/mono_3_of_4.png and /dev/null differ
diff --git a/g3doc/tutorial/images/mono_4_of_4.png b/g3doc/tutorial/images/mono_4_of_4.png
deleted file mode 100644
index 675b636..0000000
Binary files a/g3doc/tutorial/images/mono_4_of_4.png and /dev/null differ
diff --git a/g3doc/tutorial/images/pwl_calibration_distance.png b/g3doc/tutorial/images/pwl_calibration_distance.png
deleted file mode 100644
index 4bcc336..0000000
Binary files a/g3doc/tutorial/images/pwl_calibration_distance.png and /dev/null differ
diff --git a/g3doc/tutorial/images/pwl_calibration_price.png b/g3doc/tutorial/images/pwl_calibration_price.png
deleted file mode 100644
index f1ac7a2..0000000
Binary files a/g3doc/tutorial/images/pwl_calibration_price.png and /dev/null differ
diff --git a/g3doc/tutorial/index.md b/g3doc/tutorial/index.md
deleted file mode 100644
index 00dccd1..0000000
--- a/g3doc/tutorial/index.md
+++ /dev/null
@@ -1,883 +0,0 @@
-
-# TensorFlow Lattice: Lattice modeling in TensorFlow
-
-__TensorFlow Lattice__ is a library that implements lattice based models which
-are fast-to-evaluate and interpretable (optionally monotonic) models, also known
-as __interpolated look-up tables__. It includes a collection of [TensorFlow
-Lattice Estimators](#tensorflow-lattice-estimators-walk-through), which you can
-use like any [TensorFlow
-Estimator](https://www.tensorflow.org/guide/estimators), and it also
-includes lattices and piecewise linear calibration as layers that can be
-composed into custom models.
-
-Note that __TensorFlow Lattice is not an official Google product__.
-
-[TOC]
-
---------------------------------------------------------------------------------
-
-## Concepts
-
-This section is a simplified version of the description in [Monotonic Calibrated
-Interpolated Look-Up Tables](http://jmlr.org/papers/v17/15-243.html))
-
-### Lattices
-
-A __lattice__ is an interpolated look-up table that can approximate arbitrary
-input-output relationships in your data. It overlaps a regular grid on your
-input space, and it learns values for the output in the vertices of the grid.
-For a test point $$x$$, $$f(x)$$ is linearly interpolated from the lattice
-values surrounding $$x$$.
-
-
-
-The above simple example is a function with just 2 features, and has 4
-parameters: 0, 0.2, 0.4, and 1, which are the function's values at the corners
-of the input space; the rest of the function is interpolated from these
-parameters.
-
-The function $$f(x)$$ can capture non-linear interactions between features. You
-can think of the lattice parameters as the height of poles set in the ground on
-a regular grid, and the resulting function is like cloth pulled tight against
-the four poles.
-
-With $$D$$ features, a regular lattice will have $$2^D$$ parameters. To fit a
-more flexible function, you can specify a finer-grained lattice over the feature
-space. Combined with an efficient $$O(D log(D))$$ interpolation, lattice
-regression gives you __fast evaluation times__ and __arbitrarily complex
-functions__.
-
-Lattice regression functions are continuous, and piecewise infinitely
-differentiable, but they are generally not analytically differentiable at the
-lattice vertices themselves. Still, they tend to be __very smooth__.
-
-### Calibration
-
-Let's say the preceding sample lattice represents a learned *user happiness*
-with a suggestion of a coffee shop. Furthermore, assume the following:
-
-* *feature 1* is a baseline coffee price.
-* *feature 2* is the distance to a local coffee shop.
-
-We want our model to learn user happiness with a coffee shop suggestion. The
-distance can be defined from 0km to 30km and baseline coffee price can be
-something from $0 to $20.
-
-TensorFlow Lattice models use __piecewise linear functions__ to calibrate (or
-_normalize_) your input features to the range accepted by the lattice: from
-$$0.0$$ to $$1.0$$ in the example lattice above.
-
-The following diagrams show examples of what could be the calibration of the
-price and baseline coffee price using 10 keypoints each:
-
-
-
-
-
-
-All TensorFlow Lattice pre-made models (`Estimator`'s) use calibration of the
-features: the input (the $$x$$ axis of the plot above) is set to the quantiles
-(so data will be +/- evenly distributed on the keypoints), and the output ($$y$$
-axis) is learned along with the lattice(s).
-
-Notice that the calibration also handles the negative correlation of distance
-and _user happiness_.
-
-### Ensembles
-
-If you have $$D$$ features in a lattice, the number of parameters (vertices) of
-the lattice will be at least $$2^D$$. (To be precise replace 2s with the size of
-the grid for each feature.) As you can see lattices don't scale well with the
-number of features.
-
-TensorFlow Lattice offers ensembles of lattices to overcome this limitation.
-That is, several "tiny" lattices are combined (summed), enabling the model to
-grow linearly on the number of features, albeit exponential on the number of
-features in each of these "tiny" lattices, but the number of features per
-lattice are typically configured to be small.
-
-The library provides two variations of these ensembles:
-
-* __Random Tiny Lattices__ (__RTL__ for short): an arbitrary number of
- lattices of dimension $$D_l$$, each including random $$D_l$$ features out of
- the total $$D$$ input features.
-
-* __Ensembled Tiny Lattices__ (__ETL__ for short): As with RTLs, an arbitrary
- number of lattices of dimension $$D_l$$ is selected, but the input for these
- lattices are linear combinations (initialized randomly) of all the $$D$$
- inputs. It is more flexible than *RTL*, but less interpretable and may take
- longer to train.
-
---------------------------------------------------------------------------------
-
-## Why TensorFlow Lattice ?
-
-You can find a brief introduction to TensorFlow Lattice in [Google's Research
-Blog post](https://research.googleblog.com/).
-
-* __Interpretability__: the parameters of the model are the output at the
- vertices.
-
-* Powerful: __abritrarily complex__ functions with __fast evaluation times__
- (in comparison to some equivalent Deep Neural Networks for instance).
-
-As shown in the following figure, in real world usage, the training data is
-often a somewhat biased representation of where the model will be applied:
-
-
-
-TensorFlow Lattice provides the following types of __"semantic
-regularization"__:
-
-* Lattice resolution: the number of vertices in your lattice allows control
- over the flexibility of the functions that can be learned.
-
-* __Monotonicity__: You can specify that the output should only
- increase/decrease with respect to an input. In our example, you may want to
- specify that increased distance to a coffee shop should only decrease the
- chances of the coffee shop being a good one. (See the illustration below.)
-
-* __Graph Laplacian__: Outputs of the lattice/calibration vertices/keypoints
- are regularized torwards the values of their respective neighbors. So
- corners (vertices) of the space that sees less training data will fit snugly
- with the neighbors.
-
-* __Torsion__: Outputs of the lattice will be regularized towards preventing
- torsion among the features. In other words, the model will be regularized
- towards the contributions of the features being independent of each other.
-
-
-
-
-
-
-
-
-
-
-
---------------------------------------------------------------------------------
-
-## TensorFlow Lattice Estimators Walk-through
-
-TensorFlow Lattice library provides generic models formatted as pre-made
-[estimators](https://www.tensorflow.org/guide/estimators), which we
-hope will cover the typical use cases, or serve as example for those creating
-their own models.
-
-This section provides a walk-through of how to use the pre-made estimators to
-train a classifier of [Census income
-dataset](https://archive.ics.uci.edu/ml/datasets/Census+Income) using TensorFlow
-Lattice. The full code used in this section, which includes some more details,
-is in
-[examples/uci_census.py](https://github.com/tensorflow/lattice/blob/master/examples/uci_census.py).
-
-If you have trouble with the 'tf.estimator' interface, consider going over the
-[TensorFlow Linear Model Tutorial](https://www.tensorflow.org/tutorials/wide).
-All of the data parsing and formatting is very similar.
-
-### UCI Census Income Dataset
-
-For this walk-through, we will use the [UCI Census Income
-Dataset](https://archive.ics.uci.edu/ml/datasets/Census+Income). You can
-download the CSV train and test files directly from these links:
-
-* [adult.data](https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data)
-* [adult.test](https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.test)
-
-Please save the datasets into a temporary directory (for example,
-`/tmp/uci_census`) and change the `--test` and `--train` flags to point to the
-files when running the code that follows.
-
-The data is available as CSV, and we use [pandas data analysis
-library](http://pandas.pydata.org/) (`pip install pandas` on most platforms,
-maybe requiring `sudo`) to make the parsing easy.
-
-The `tf.estimator` models use an input builder function, usually named
-`input_fn` which is reponsible to parse data and convert into `tf.Tensor`s (or
-`tf.SparseTensor`s) with batches of data.
-
-Our `input_fn` functions look like the following:
-
-```python
-import pandas as pd
-import tensorflow as tf
-import tensorflow_lattice as tfl
-
-flags = tf.flags
-FLAGS = flags.FLAGS
-
-flags.DEFINE_string("test", "/tmp/uci_census/adult.test", "Path to test file.")
-flags.DEFINE_string("train", "/tmp/uci_census/adult.data", "Path to train file.")
-
-CSV_COLUMNS = [
- "age", "workclass", "fnlwgt", "education", "education_num",
- "marital_status", "occupation", "relationship", "race", "gender",
- "capital_gain", "capital_loss", "hours_per_week", "native_country",
- "income_bracket"
-]
-
-def get_test_input_fn(batch_size, num_epochs, shuffle):
- return get_input_fn(FLAGS.test, batch_size, num_epochs, shuffle)
-
-
-def get_train_input_fn(batch_size, num_epochs, shuffle):
- return get_input_fn(FLAGS.train, batch_size, num_epochs, shuffle)
-
-
-def get_input_fn(file_path, batch_size, num_epochs, shuffle):
- df_data = pd.read_csv(
- tf.gfile.Open(file_path),
- names=CSV_COLUMNS,
- skipinitialspace=True,
- engine="python",
- skiprows=1)
- df_data = df_data.dropna(how="any", axis=0)
- labels = df_data["income_bracket"].apply(lambda x: ">50K" in x).astype(int)
- return tf.estimator.inputs.pandas_input_fn(
- x=df_data,
- y=labels,
- batch_size=batch_size,
- shuffle=shuffle,
- num_epochs=num_epochs,
- num_threads=1)
-```
-
-### Preparing `FeatureColumns`
-
-TensorFlow provides `FeatureColumn`s as a way to select and describe the
-features used for a model. Numeric features require no transformations; we need
-to list the known valid values of categorical features.
-
-See more details in [TensorFlow Linear Model
-tutorial](https://www.tensorflow.org/tutorials/wide).
-
-TensorFlow Lattice pre-made estimators will take any of the currently supported
-`FeatureColumns` or alternatively the raw columns coming from the `input_fn`
-function, if they are properly numeric already.
-
-```python
-def get_feature_columns():
- # Categorical features.
- gender =
- tf.feature_column.categorical_column_with_vocabulary_list(
- "gender", ["Female", "Male"])
- education =
- tf.feature_column.categorical_column_with_vocabulary_list(
- "education", [
- "Bachelors", "HS-grad", "11th", "Masters", "9th", "Some-college",
- "Assoc-acdm", "Assoc-voc", "7th-8th", "Doctorate", "Prof-school",
- "5th-6th", "10th", "1st-4th", "Preschool", "12th"
- ])
- …
- # Numerical (continuous) base columns.
- age = tf.feature_column.numeric_column("age")
- education_num = tf.feature_column.numeric_column("education_num")
- capital_gain = tf.feature_column.numeric_column("capital_gain")
- …
- return [
- age,
- workclass,
- education,
- education_num,
- marital_status,
- occupation,
- relationship,
- race,
- gender,
- capital_gain,
- capital_loss,
- hours_per_week,
- native_country,
- ]
-
-```
-
-Note: unlike DNN pre-made estimators
-([DNNClassifier](https://www.tensorflow.org/versions/r1.3/api_docs/python/tf/estimator/DNNClassifier)
-and
-[DNNRegressor](https://www.tensorflow.org/versions/r1.3/api_docs/python/tf/estimator/DNNClassifier)),
-TensorFlow Lattice pre-made estimators accept sparse `FeatureColumn` without the
-need for embedding them.
-
-### Calibration: Saving The Quantiles
-
-TensorFlow Lattice requires proper calibration of the input for its lattices
-(see section on [calibration](#calibration) above).
-
-The current default calibration algorithm requires quantiles information about
-the data on which it's going to train. This can be done as a simple
-pre-processing step.
-
-The following code snippet from our example does that:
-
-```python
-import tensorflow_lattice as tfl
-
-flags.DEFINE_bool("create_quantiles", False,
- "Run once to create histogram of features for calibration.")
-flags.DEFINE_string(
- "quantiles_dir", None,
- "Directory where to store quantile information, defaults to the model "
- "directory (set by --output-dir) but since quantiles can be reused by "
- "models with different parameters, you may want to have a separate "
- "directory.")
-…
-
-def create_quantiles(quantiles_dir):
- """Creates quantiles directory if it doesn't yet exist."""
- batch_size = 10000
- input_fn = get_test_input_fn(
- batch_size=batch_size, num_epochs=1, shuffle=False)
- # Reads until input is exhausted, 10000 at a time.
- tfl.save_quantiles_for_keypoints(
- input_fn=input_fn,
- save_dir=quantiles_dir,
- feature_columns=create_feature_columns(),
- num_steps=None)
-
-def main(argv)
- …
-
- # Create quantiles if required.
- if FLAGS.create_quantiles:
- if FLAGS.run != "train":
- raise ValueError(
- "Can not create_quantiles for mode --run='{}'".format(FLAGS.run))
- create_quantiles(quantiles_dir)
-```
-
-Note: This only needs to be run once per dataset, and can be shared among
-different models that use the same data.
-
-Note: This information is only needed for training. During inference
-(production), the model itself already contains all the information it needs,
-and doesn't need to read this anymore.
-
-Advanced: If you know the range of input, instead of using quantiles, you can
-provide
-[`uniform_keypoints_for_signal`](../api_docs/python/tensorflow_lattice/uniform_keypoints_for_signal.md)
-as function initializer, which will create calibration keypoints uniformly in
-the given range. Or you can provide your own keypoint initializing function.
-
-### Calibrated Linear Model
-
-Calibrated linear model is the simplest model type offered in TensorFlow
-Lattice. It calibrates the input using piecewise-linear calibrated functions and
-then linearly combine the inputs. Using it is trivial, if you are used to
-TensorFlow's `Estimator` framework (see [Module
-tf.estimator](https://www.tensorflow.org/api_docs/python/tf/estimator)).
-
-To create a calibrated linear model, you need to specify features in
-`feature_columns`, the model directory in `model_dir`, the ["run
-configuration"](https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig)
-in `config`, and in `hparams` the hyperparameters settings in the form of a
-[`tfl.CalibratedLinearHParams`](../api_docs/python/tensorflow_lattice/CalibratedLinearHParams.md)
-object.
-
-All parameters are optional; see more details in:
-
-* [`tfl.calibrated_linear_classifier`](../api_docs/python/tensorflow_lattice/calibrated_linear_classifier.md)
-* [`tfl.calibrated_linear_regressor`](../api_docs/python/tensorflow_lattice/calibrated_linear_regressor.md)
-* Configurable hyperparameters in
- [`tfl.CalibratedLinearHParams`](../api_docs/python/tensorflow_lattice/CalibratedLinearHParams.md)
-
-Calibration can be forced to be monotonic and regularized in different ways. It
-also supports special casing of __missing values__ (see `missing_input_value`
-hyperparameter); that is, the calibration of missing values has its own
-parameter that is learned independently from other values.
-
-An example of code that stitches this together is presented below. For now we
-present only the default hyperparameters. The next section covers the special
-TensorFlow Lattice hyperparameters, and how to change them.
-
-The following code shows how our `create_calibrated_linear` function gets
-called. It hinges on creating an `Estimator` object, and then either training or
-evaluating it.
-
-```python
-import tensorflow_lattice as tfl
-
-def create_calibrated_linear(feature_columns, config, quantiles_dir):
- feature_names = [fc.name for fc in feature_columns]
- hparams = tfl.CalibratedLinearHParams(feature_names=feature_names)
- return tfl.calibrated_linear_classifier(
- feature_columns=feature_columns,
- model_dir=config.model_dir,
- config=config,
- hparams=hparams,
- quantiles_dir=quantiles_dir)
-…
-
-def create_estimator(config, quantiles_dir):
- """Creates estimator for given configuration based on --model_type."""
- feature_columns = create_feature_columns()
- if FLAGS.model_type == "calibrated_linear":
- return create_calibrated_linear(feature_columns, config, quantiles_dir)
- elif FLAGS.model_type == "calibrated_lattice":
- return create_calibrated_lattice(feature_columns, config, quantiles_dir)
- elif FLAGS.model_type == "calibrated_rtl":
- return create_calibrated_rtl(feature_columns, config, quantiles_dir)
- elif FLAGS.model_type == "calibrated_etl":
- return create_calibrated_etl(feature_columns, config, quantiles_dir)
- elif FLAGS.model_type == "calibrated_dnn":
- return create_calibrated_dnn(feature_columns, config, quantiles_dir)
-
- raise ValueError("Unknown model_type={}".format(FLAGS.model_type))
- …
-
-def main(args):
- …
- # Create config and then model.
- config = tf.estimator.RunConfig().replace(model_dir=output_dir)
- estimator = create_estimator(config, quantiles_dir)
-
- if FLAGS.run == "train":
- train(estimator)
-
- elif FLAGS.run == "evaluate":
- evaluate(estimator)
-
- else:
- raise ValueError("Unknonw --run={}".format(FLAGS.run))
-```
-
-### Hyperparameters setting
-
-Each of the pre-made estimators offered by *TensorFlow Lattices* is controlled
-by a set of hyperparameters. Some are shared among different estimators, some
-are unique. All are documented in their definition.
-
-* Calibrated linear models:
- [`tfl.CalibratedLinearHParams`](../api_docs/python/tensorflow_lattice/CalibratedLinearHParams.md)
-* Calibrated lattice models:
- [`tfl.CalibratedLatticeHParams`](../api_docs/python/tensorflow_lattice/CalibratedLatticeHParams.md)
-* Calibrated RTL models:
- [`tfl.CalibratedRtlHParams`](../api_docs/python/tensorflow_lattice/CalibratedRtlHParams.md)
-* Calibrated ETL models:
- [`tfl.CalibratedEtlHParams`](../api_docs/python/tensorflow_lattice/CalibratedEtlHParams.md)
-
-TensorFlow Lattices' hyperparameters classes are slightly different from
-[TensorFlow standard hyperparameters
-class](https://www.tensorflow.org/api_docs/python/tf/contrib/training/HParams)
-in that they accept global and per-feature parameters. For instance, in our
-calibrated linear model on the previous section, we defined the following
-default values:
-
-```python
-def create_calibrated_linear(feature_columns, config, quantiles_dir):
- feature_names = [fc.name for fc in feature_columns]
- hparams = tfl.CalibratedLinearHParams(
- feature_names=feature_names, num_keypoints=200, learning_rate=0.1)
- hparams.set_feature_param("capital_gain", "calibration_l2_laplacian_reg",
- 4.0e-8)
- hparams.parse(FLAGS.hparams)
- _pprint_hparams(hparams)
- return tfl.calibrated_linear_classifier(
- feature_columns=feature_columns,
- model_dir=config.model_dir,
- config=config,
- hparams=hparams,
- quantiles_dir=quantiles_dir)
-```
-
-The preceding code uses different default values for the following
-hyperparameters:
-
-* `num_keypoints`
-* `learning_rate`
-* `calibration_l2_laplacian_reg` (l2 laplacian regularization for the
- calibration) for the feature named "capital_gain"
-
-Notice also the call `hparams.parse(FLAGS.hparams)`, which will parse a string
-with a comma-separated list of settings. Feature specific values can also be set
-by prefixing the parameter (that takes feature specific values) with
-"feature\_\_<_feature\_name_>\_\_<_param\_name_>". Notice that the
-separator here is double underscores ("\_\_").
-
-For example, the following invocation sets `learning_rate=0.001` and for feature
-_capital\_loss_ it sets `calibration_l2_laplacian_reg=1.0e-5`:
-
-```bash
-
-$ uci_census.py … --hparams=learning_rate=0.001,feature__capital_loss__calibration_l2_laplacian_reg=1.0e-5 …
-
-```
-
-We define this simple pretty-print function in our example:
-
-```python
-def _pprint_hparams(hparams):
- """Pretty-print hparams."""
- print("* hparams=[")
- for (key, value) in sorted(six.iteritems(hparams.values())):
- print("\t{}={}".format(key, value))
- print("]")
-```
-
-### Calibrated Lattice Model
-
-Calibrated lattice models first calibrate the input with piecewise-linear
-functions, and combine them into a lattice (see [Concepts](#concepts) section).
-
-Calibrated lattice models also provide:
-
-* __Enforced monotonicity__: in the calibration (can be increasing or
- decreasing), and in the lattice (you must also set the calibration to be
- monotonic and enable lattice monotonicity). Both can be selected per
- feature.
-* __Missing value handle__: missing values can be calibrated automatically for
- some special value or can have their own value in the lattice. Controlled
- per feature through the parameters: `missing_input_value` and
- `missing_vertex`.
-* __Semantic regularization__: rich set of regularization that can be applied
- independently to the calibration and lattice. Can be set globally or per
- feature. See their description in the [Concepts](#concepts) section.
-* Flexible size: lattice can easily be adjusted to different granularity per
- feature by setting `lattice_size`. This allows it lots of power to
- aproximate any function.
-
-Limitations:
-
-* __Scalability issues on number of features and lattice size__: the total
- number of vertices (parameters) in the lattice is the product of the
- `lattice_size` for each feature. Your models are gated by available memory
- and parameters update speed. To stay within reasonable bounds, don't use
- more than 14 features (or 50,000 parameters). If that isn't possible, use
- the more powerful [Random Tiny Lattices Model](#random-tiny-lattices-model)
- or [Embedded Tiny Lattices Model](#embedded-tiny-lattices-model).
-
-Calibrated lattice models are available as classifier or regressor by
-[`tfl.calibrated_lattice_classifier`](../api_docs/python/tensorflow_lattice/calibrated_lattice_classifier.md)
-and
-[`tfl.calibrated_lattice_regressor`](../api_docs/python/tensorflow_lattice/calibrated_lattice_regressor.md)
-constructors.
-
-Documentation on all hyperparameters is in
-[`tfl.CalibratedLatticeHParams`](../api_docs/python/tensorflow_lattice/CalibratedLatticeHParams.md)
-
-Extract from the
-[`uci_census`](https://github.com/tensorflow/lattice/blob/master/examples/uci_census.py)
-example:
-
-```python
-def create_calibrated_lattice(feature_columns, config, quantiles_dir):
- feature_names = [fc.name for fc in feature_columns]
- hparams = tfl.CalibratedLatticeHParams(
- feature_names=feature_names,
- num_keypoints=200,
- lattice_l2_laplacian_reg=5.0e-3,
- lattice_l2_torsion_reg=1.0e-4,
- learning_rate=0.1,
- lattice_size=2)
- hparams.parse(FLAGS.hparams)
- _pprint_hparams(hparams)
- return tfl.calibrated_lattice_classifier(
- feature_columns=feature_columns,
- model_dir=config.model_dir,
- config=config,
- hparams=hparams,
- quantiles_dir=quantiles_dir)
-```
-
-Note: To see how this function gets called from `main`, see [Calibrated Linear
-Model](#calibrated-linear-model).
-
-### Random Tiny Lattices Model
-
-Calibrated "Random Tiny Lattices" (RTL) models, like calibrated lattice models,
-first calibrate the input with piecewise-linear functions. But then it combines
-them in an ensemble of `num_lattices` lattices built with inputs from random
-features (`lattice_rank` input features per lattice).
-
-Extract from the
-[`uci_census`](https://github.com/tensorflow/lattice/blob/master/examples/uci_census.py)
-example:
-
-```python
-def create_calibrated_rtl(feature_columns, config, quantiles_dir):
- feature_names = [fc.name for fc in feature_columns]
- hparams = tfl.CalibratedRtlHParams(
- feature_names=feature_names,
- num_keypoints=200,
- learning_rate=0.02,
- lattice_l2_laplacian_reg=5.0e-4,
- lattice_l2_torsion_reg=1.0e-4,
- lattice_size=3,
- lattice_rank=4,
- num_lattices=100)
- # Specific feature parameters.
- hparams.set_feature_param("capital_gain", "lattice_size", 8)
- hparams.set_feature_param("native_country", "lattice_size", 8)
- hparams.set_feature_param("marital_status", "lattice_size", 4)
- hparams.set_feature_param("age", "lattice_size", 8)
- hparams.parse(FLAGS.hparams)
- _pprint_hparams(hparams)
- return tfl.calibrated_rtl_classifier(
- feature_columns=feature_columns,
- model_dir=config.model_dir,
- config=config,
- hparams=hparams,
- quantiles_dir=quantiles_dir)
-```
-
-Note: To see how this function gets called from `main`, see [Calibrated Linear
-Model](#calibrated-linear-model).
-
-In this example it will calibrate the inputs (using up to 200 keypoints, per
-`num_keypoints`) and then randomly distribute them into 100 lattices
-(`num_lattices`, a feature can be used by more than one lattice).
-
-The lattices and the calibration are all trained jointly.
-
-Like with calibrated lattice models, but without the limitations on the number
-of features, it supports:
-
-* __Enforced monotonicity__: in the calibration (can be increasing or
- decreasing), and in the lattice (one must also set the calibration to be
- monotonic, and enable lattice monotonicity). Both can be selected per
- feature.
-* __Missing value handle__: Missing values can be calibrated automatically for
- some special value, or can have their own value in the lattice. Controlled
- per feature through the parameters: `missing_input_value` and
- `missing_vertex`.
-* __Semantic regularization__: rich set of regularization that can be applied
- independently to the calibration and lattice. Can be set globally or per
- feature. See their description in the [Concepts](#concepts) section.
-* Flexible size: lattice can easily be adjusted to different granularity per
- feature by setting `lattice_size`. This allows it lots of power to
- aproximate any function.
-
-Note: The `lattice_rank` hyperparameter controls how many features are seen in
-_combination_. It is often used as a regularization on the complexity of
-interactions allowed among the features. But as with calibrated lattices this is
-limited to 10 or 20 features at most combined in the same lattices. If you
-wonder if the model could pick better than random features to be combined in
-lattices, check out the next session, on [Embedded Tiny Lattices
-Model](#embedded-tiny-lattices)
-
-Calibrated RTL models are available as classifier or regressor by
-[`tfl.calibrated_rtl_classifier`](../api_docs/python/tensorflow_lattice/calibrated_rtl_classifier.md)
-and
-[`tfl.calibrated_rtl_regressor`](../api_docs/python/tensorflow_lattice/calibrated_rtl_regressor.md)
-constructors.
-
-Documentation on all hyperparameters in
-[`tfl.CalibratedLatticeHParams`](../api_docs/python/tensorflow_lattice/CalibratedLatticeHParams.md)
-
-Note: See above in section [Calibrated Linear Model](#calibrated-linear-model)
-on how this function gets called from `main`.
-
-### Embedded Tiny Lattices Model
-
-Calibrated "Embedded Tiny Lattices" (ETL) models, like calibrated [RTL
-models](#random-tiny-lattices-model), first calibrate the input and connect
-those calibrated signals into an ensemble of lattices. But as opposed to have
-each lattice take as input a subset of the calibrated features, in ETL models it
-takes as input an embedding of the input features: each input is a linear
-combination of the calibrated features.
-
-The number of lattices is defined by 'monotonic_num_lattices' and
-'non_monotonic_num_lattices': monotonic lattices can take as input monotonic
-features and non-monotonic features. Non-monotonic lattices can only take
-non-monotonic features as input (otherwise monotonicity could be broken).
-
-The size of the embedding to be used in each lattice is given by
-`monotonic_lattice_rank` and `non_monotonic_lattice_rank`. Each lattice has it's
-own embedding: calibration, embedding and lattices are trained jointly.
-
-The size of the lattices, which gives resolution for them is given by
-`monotonic_lattice_size` and `non_monotonic_lattice_size`.
-
-Calibrated ETL models are available as classifier or regressor by
-[`tfl.calibrated_etl_classifier`](../api_docs/python/tensorflow_lattice/calibrated_etl_classifier.md)
-and
-[`tfl.calibrated_etl_regressor`](../api_docs/python/tensorflow_lattice/calibrated_etl_regressor.md)
-constructors.
-
-Embedded tiny lattices can be __more powerful__ than [RTL
-models](#random-tiny-lattices-model), but they __sacrifice some of the "semantic
-regularization"__ (same regularization options are available, but they apply to
-abstract embeddings), and are __slower to train__. __Monotonicity still is well
-supported__.
-
-See details in paper [Deep Lattice Networks and Partial Monotonic
-Functions](https://research.google.com/pubs/pub46327.html). ETL implements only
-one layer deep lattice models, but deeper models can be built by composing
-lattice layers, in the [next session](#tensorflow-lattice-layers)
-
-In our example
-[`uci_census`](https://github.com/tensorflow/lattice/blob/master/examples/uci_census.py)
-model, using only non-monotonic signals:
-
-```python
-def create_calibrated_etl(feature_columns, config, quantiles_dir):
- # No enforced monotonicity in this example.
- feature_names = [fc.name for fc in feature_columns]
- hparams = tfl.CalibratedEtlHParams(
- feature_names=feature_names,
- num_keypoints=200,
- learning_rate=0.02,
- non_monotonic_num_lattices=200,
- non_monotonic_lattice_rank=2,
- non_monotonic_lattice_size=2,
- calibration_l2_laplacian_reg=4.0e-3,
- lattice_l2_laplacian_reg=1.0e-5,
- lattice_l2_torsion_reg=4.0e-4)
- hparams.parse(FLAGS.hparams)
- _pprint_hparams(hparams)
- return tfl.calibrated_etl_classifier(
- feature_columns=feature_columns,
- model_dir=config.model_dir,
- config=config,
- hparams=hparams,
- quantiles_dir=quantiles_dir)
-```
-
-Note: To see how this function gets called from `main`, see [Calibrated Linear
-Model](#calibrated-linear-model).
-
---------------------------------------------------------------------------------
-
-## TensorFlow Lattice Layers
-
-TensorFlow Lattice layer components are also provided by the library, so users
-can combine them in more flexible or advanced ways.
-
-The following are the layer components included in the TensorFlow Lattice
-library:
-
-* __Piecewise-Linear Calibration__:
- * [`tfl.input_calibration_layer`](../api_docs/python/tensorflow_lattice/input_calibration_layer.md):
- Calibrates the "input", provided either as `FeatureColumn`s or as a dict
- of columns to tensors, the typical object returned by an `input_fn`
- function. Includes support for monotonicity, regularization and special
- "missing" values.
- * [`tfl.input_calibration_layer_from_hparams`](../api_docs/python/tensorflow_lattice/input_calibration_layer_from_hparams.md):
- Calibrates the "input", provided either as `FeatureColumn`s or as a dict
- of columns to tensors, the typical object returned by an `input_fn`
- function. Includes support for monotonicity, regularization and special
- "missing" values. This version uses an `tfl.CalibrateHParams` to specify
- the hyperparameters.
- * [`tfl.calibration_layer`](../api_docs/python/tensorflow_lattice/calibration_layer.md):
- Calibrates a tensor of shape \[batch_size, ...\]. Each element (outside
- the batch-dimension) gets its own calibration. Includes support for
- monotonicity, regularization and special "missing" values.
-* __Lattice Layer__:
- * [`tfl.lattice_layer`](../api_docs/python/tensorflow_lattice/lattice_layer.md):
- Creates `output_dim` lattices that uses as input a tensor of shape
- \[batch_size, input_dim\]. Lattice size is defined for each dimension of
- `input_dim`. The total number of parameters is the product of all these
- lattice sizes times `output_dim`. Full support of monotonicity and
- regularization.
- * [`tfl.ensemble_lattices_layer`](../api_docs/python/tensorflow_lattice/ensemble_lattices_layer.md):
- Creates an ensemble of lattices connecting inputs as specified by the
- caller. Full support of monotonicity and regularization.
-
-Example *calibrated_dnn*, a custom estimator from our example
-[`uci_census`](https://github.com/tensorflow/lattice/blob/master/examples/uci_census.py)
-model:
-
-```python
-def create_calibrated_dnn(feature_columns, config, quantiles_dir):
- """Creates a calibrated DNN model."""
- # This is an example of a hybrid model that uses input calibration layer
- # offered by TensorFlow Lattice library and connects it to DNN.
- feature_names = [fc.name for fc in feature_columns]
- hparams = tfl.CalibratedHParams(
- feature_names=feature_names,
- num_keypoints=200,
- learning_rate=1.0e-3,
- calibration_output_min=-1.0,
- calibration_output_max=1.0,
- nodes_per_layer=10, # All layers have the same number of nodes.
- layers=2, # Includes output layer, therefore >= 1.
- )
- hparams.parse(FLAGS.hparams)
- _pprint_hparams(hparams)
-
- def _model_fn(features, labels, mode, params):
- """Model construction closure used when creating estimator."""
- del mode
- del params # They are read directly from the bound variable hparams
-
- # Calibrate: since there is no monotonicity, there are no projection ops.
- # We also discard the ordered names of the features.
- (output, _, _, regularization) = tfl.input_calibration_layer_from_hparams(
- features, feature_columns, hparams, quantiles_dir)
-
- # Hidden-layers.
- for _ in range(hparams.layers - 1):
- output = tf.layers.dense(
- inputs=output, units=hparams.nodes_per_layer, activation=tf.sigmoid)
-
- # Classifier logits and prediction.
- logits = tf.layers.dense(inputs=output, units=1)
- predictions = tf.reshape(tf.sigmoid(logits), [-1])
-
- # Notice loss doesn't include regularization, which is added separately
- # by means of tf.contrib.layers.apply_regularization().
- loss_no_regularization = tf.losses.log_loss(labels, predictions)
- loss = loss_no_regularization
- if regularization is not None:
- loss += regularization
- optimizer = tf.train.AdamOptimizer(learning_rate=hparams.learning_rate)
- train_op = optimizer.minimize(
- loss,
- global_step=tf.train.get_global_step(),
- name="calibrated_dnn_minimize")
-
- eval_metric_ops = {
- "accuracy": tf.metrics.accuracy(labels, predictions),
-
- # We want to report the loss without the regularization, so metric is
- # comparable with different regularizations. FutureWork, list both.
- "average_loss": tf.metrics.mean(loss_no_regularization),
- }
-
- return tf.estimator.EstimatorSpec(mode, predictions, loss, train_op,
- eval_metric_ops)
-
- # Hyperparameters are passed directly to the model_fn closure by the context.
- return tf.estimator.Estimator(
- model_fn=_model_fn,
- model_dir=config.model_dir,
- config=config,
- params=None)
-```
-
-### Other potential use cases of these components
-
-* If integrating an embedding from another model (transfer-learning);
-* Use TensorFlow Lattice\'s calibration in a DNN: works much better than
- gaussian normalization of inputs. Something else that has been used with
- some success is the piecewise linear function as an activation function.
-* Use lattices on the "upper" (closer to output) layers of a DNN, for its
- regularization.
-* Use the piecewise-linear calibration as an activation function for neural
- networks.
-* Use piecewise-linear calibration as a probability distribution function for
- learning continuous values in a Reinforcement Learning set up (REINFORCE
- algorithm).
-
-## Papers
-
-* [Lattice Regression](https://papers.nips.cc/paper/3694-lattice-regression),
- Eric Garcia, Maya Gupta, Advances in Neural Information Processing Systems
- (NIPS), 2009
-* [Optimized Regression for Efficient Function
- Evaluation](http://ieeexplore.ieee.org/document/6203580/), Eric Garcia,
- Raman Arora, Maya R. Gupta, IEEE Transactions on Image Processing, 2012
-* [Monotonic Calibrated Interpolated Look-Up
- Tables](http://jmlr.org/papers/v17/15-243.html), Maya Gupta, Andrew Cotter,
- Jan Pfeifer, Konstantin Voevodski, Kevin Canini, Alexander Mangylov,
- Wojciech Moczydlowski, Alexander van Esbroeck, Journal of Machine Learning
- Research (JMLR), 2016
-* [Fast and Flexible Monotonic Functions with Ensembles of
- Lattices](https://papers.nips.cc/paper/6377-fast-and-flexible-monotonic-functions-with-ensembles-of-lattices),
- Mahdi Milani Fard, Kevin Canini, Andrew Cotter, Jan Pfeifer, Maya Gupta,
- Advances in Neural Information Processing Systems (NIPS), 2016
-* [Deep Lattice Networks and Partial Monotonic
- Functions](https://research.google.com/pubs/pub46327.html), Seungil You,
- Kevin Canini, David Ding, Jan Pfeifer, Maya R. Gupta, Advances in Neural
- Information Processing Systems (NIPS), 2017
diff --git a/pip_pkg.sh b/pip_pkg.sh
deleted file mode 100755
index dc483a0..0000000
--- a/pip_pkg.sh
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-set -e
-
-PLATFORM="$(uname -s | tr 'A-Z' 'a-z')"
-
-function main() {
- if [ $# -lt 1 ] ; then
- echo "No destination dir provided"
- exit 1
- fi
-
- # Create the directory, then do dirname on a non-existent file inside it to
- # give us an absolute paths with tilde characters resolved to the destination
- # directory. Readlink -f is a cleaner way of doing this but is not available
- # on a fresh macOS install.
- mkdir -p "$1"
- DEST="$(dirname "${1}/does_not_exist")"
- echo "=== destination directory: ${DEST}"
-
- TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXX)
-
- echo $(date) : "=== Using tmpdir: ${TMPDIR}"
-
- echo "=== Copy TensorFlow Lattice files"
- # Here are bazel-bin/pip_pkg.runfiles directory structure.
- # bazel-bin/pip_pkg.runfiles
- # |-
- # |- org_python_pypi_backports_weakref
- # |- org_tensorflow
- # |- protobuf
- # |- six_archive
- # |- tensorflow_lattice
- # |- external
- # |- pip_pkg
- # |- pip_pkg.sh
- # |- MANIFEST.in (needed)
- # |- setup.py (needed)
- # |- tensorflow_lattice (needed)
- #
- # To build tensorflow lattice wheel, we only need setup.py, MANIFEST.in, and
- # python and .so files under tensorflow_lattice/tensorflow_lattice.
- # So we extract those to ${TMPDIR}.
- cp bazel-bin/pip_pkg.runfiles/tensorflow_lattice/setup.py "${TMPDIR}"
- cp bazel-bin/pip_pkg.runfiles/tensorflow_lattice/MANIFEST.in "${TMPDIR}"
- cp -R \
- bazel-bin/pip_pkg.runfiles/tensorflow_lattice/tensorflow_lattice \
- "${TMPDIR}"
-
- echo "=== Copy TensorFlow Lattice root and cc files"
- cp README.md ${TMPDIR}
- cp LICENSE ${TMPDIR}
- cp -R \
- tensorflow_lattice/cc \
- "${TMPDIR}/tensorflow_lattice"
-
- pushd ${TMPDIR}
- if [ "${TFL_SDIST}" = true ]; then
- echo $(date) : "=== Building source distribution and wheel"
- else
- echo $(date) : "=== Building wheel"
- fi
-
- if [ -z "$2" ]; then
- if [ "${TFL_SDIST}" = true ]; then
- python setup.py sdist > /dev/null
- fi
- python setup.py bdist_wheel > /dev/null
- else
- if [ "${TFL_SDIST}" = true ]; then
- python setup.py "$2" sdist > /dev/null
- fi
- python setup.py "$2" bdist_wheel >/dev/null
- fi
-
- cp dist/* "${DEST}"
- popd
- rm -rf ${TMPDIR}
- echo $(date) : "=== Output tar ball and wheel file are in: ${DEST}"
-}
-
-main "$@"
diff --git a/setup.py b/setup.py
index 3e95688..cf46697 100644
--- a/setup.py
+++ b/setup.py
@@ -1,114 +1,105 @@
-# pylint: disable=g-bad-file-header
-# Copyright 2017 The TensorFlow Lattice Authors.
+# Copyright 2018 The TensorFlow Lattice Authors.
#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# =============================================================================
-"""Setup for pip package."""
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+# ==============================================================================
+"""Package setup script for TensorFlow Lattice library."""
+
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
+import datetime
import sys
-import warnings
from setuptools import find_packages
from setuptools import setup
-from setuptools.command.install import install as InstallCommandBase
-from setuptools.dist import Distribution
-
-
-__version__ = '0.9.9'
-
-
-REQUIRED_PACKAGES = [
- 'six >= 1.11.0',
- 'protobuf >= 3.6.1',
- 'numpy >= 1.14.5',
-]
+# This version number should always be that of the *next* (unreleased) version.
+# Immediately after uploading a package to PyPI, you should increment the
+# version number and push to gitHub.
+__version__ = "2.0"
-if '--gpu' in sys.argv:
- use_gpu = True
- sys.argv.remove('--gpu')
+if "--release" in sys.argv:
+ sys.argv.remove("--release")
+ _name = "tensorflow_lattice"
else:
- use_gpu = False
-
-
-if use_gpu:
- project_name = 'tensorflow-lattice-gpu'
- REQUIRED_PACKAGES.append('tensorflow-gpu==1.14.0')
-else:
- project_name = 'tensorflow-lattice'
- REQUIRED_PACKAGES.append('tensorflow==1.14.0')
-
-CONSOLE_SCRIPTS = [
- 'freeze_graph_wrapper = '
- 'tensorflow_lattice.cc.tflite.freeze_graph_wrapper:main',
- 'toco_wrapper = tensorflow_lattice.cc.tflite.toco_wrapper:main',
+ # Build a nightly package by default.
+ _name = "tensorflow_lattice_nightly"
+ __version__ += datetime.datetime.now().strftime(".dev%Y%m%d")
+
+_install_requires = [
+ "absl-py",
+ "numpy",
+ "pandas",
+ "six",
+ "sklearn",
+ "matplotlib",
+ "graphviz",
]
+# Part of the visualization code uses colabtools and IPython libraries. These
+# are not added as hard requirements as they are mainly used in jupyter/colabs.
+
+_extras_require = {
+ "tensorflow": "tensorflow>=1.15",
+ "tensorflow-gpu": "tensorflow-gpu>=1.15",
+}
+
+_classifiers = [
+ "Development Status :: 4 - Beta",
+ "Intended Audience :: Developers",
+ "Intended Audience :: Education",
+ "Intended Audience :: Science/Research",
+ "License :: OSI Approved :: Apache Software License",
+ "Operating System :: OS Independent",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 2",
+ "Programming Language :: Python :: 3",
+ "Topic :: Scientific/Engineering",
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
+ "Topic :: Scientific/Engineering :: Mathematics",
+ "Topic :: Software Development",
+ "Topic :: Software Development :: Libraries",
+ "Topic :: Software Development :: Libraries :: Python Modules",
+]
-class BinaryDistribution(Distribution):
- """This class is needed in order to create OS specific wheels."""
-
- def has_ext_modules(self):
- return True
-
-
-warnings.warn('tensorflow-lattice is likley to fail when building from a '
- 'source distribution (sdist). Please follow instructions in '
- '(https://github.com/tensorflow/lattice/INSTALL.md) '
- 'to build this from the source.')
-
+_description = (
+ "A library that implements optionally monotonic lattice based models.")
+_long_description = """\
+TensorFlow Lattice is a library that implements fast-to-evaluate and
+interpretable (optionally monotonic) lattice based models, which are also known
+as *interpolated look-up tables*. The library includes a collection of
+Estimators, which operate like any TensorFlow Estimator. It also includes
+Keras layers for lattices and feature calibration that can be composed
+into custom models.
+"""
setup(
- name=project_name,
+ name=_name,
version=__version__,
- description=('TensorFlow Lattice provides lattice models in TensorFlow'),
- long_description='',
- url='https://github.com/tensorflow/lattice',
- author='Google Inc.',
- author_email='tensorflow-lattice-releasing@google.com',
- # Contained modules and scripts.
+ author="Google Inc.",
+ author_email="no-reply@google.com",
+ license="Apache 2.0",
+ classifiers=_classifiers,
+ install_requires=_install_requires,
+ extras_require=_extras_require,
packages=find_packages(),
- install_requires=REQUIRED_PACKAGES,
- # Add in any packaged data.
include_package_data=True,
- package_data={'': ['*.so']},
- exclude_package_data={'': ['BUILD', '*.h', '*.cc']},
- zip_safe=False,
- distclass=BinaryDistribution,
- cmdclass={
- 'pip_pkg': InstallCommandBase,
- },
- entry_points={
- 'console_scripts': CONSOLE_SCRIPTS
- },
- # PyPI package information.
- classifiers=[
- 'Development Status :: 4 - Beta',
- 'Intended Audience :: Developers',
- 'Intended Audience :: Education',
- 'Intended Audience :: Science/Research',
- 'License :: OSI Approved :: Apache Software License',
- 'Programming Language :: Python :: 2.7',
- 'Programming Language :: Python :: 3.4',
- 'Programming Language :: Python :: 3.5',
- 'Programming Language :: Python :: 3.6',
- 'Topic :: Scientific/Engineering :: Mathematics',
- 'Topic :: Software Development :: Libraries :: Python Modules',
- 'Topic :: Software Development :: Libraries',
- ],
- license='Apache 2.0',
- keywords='lattice tensorflow tensor machine learning',
+ description=_description,
+ long_description=_long_description,
+ long_description_content_type="text/markdown",
+ keywords="tensorflow lattice calibration machine learning",
+ url=(
+ "https://github.com/tensorflow/lattice"
+ ),
)
diff --git a/tensorflow b/tensorflow
deleted file mode 160000
index 456fbc0..0000000
--- a/tensorflow
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 456fbc0e498e3d10604973de9f46ca48d62267cc
diff --git a/tensorflow_lattice/BUILD b/tensorflow_lattice/BUILD
index 37af0c8..835d220 100644
--- a/tensorflow_lattice/BUILD
+++ b/tensorflow_lattice/BUILD
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
-licenses(["notice"]) # Apache 2.0 License
package(
default_visibility = [
@@ -20,38 +19,31 @@ package(
],
)
+licenses(["notice"])
+
exports_files(["LICENSE"])
py_library(
name = "tensorflow_lattice",
- srcs = ["__init__.py"],
- srcs_version = "PY2AND3",
- deps = [
- "//tensorflow_lattice/python:keypoints_initialization",
- "//tensorflow_lattice/python:lattice_layers",
- "//tensorflow_lattice/python:lattice_ops_py",
- "//tensorflow_lattice/python:pwl_calibration_layers",
- "//tensorflow_lattice/python:pwl_calibration_ops_py",
- "//tensorflow_lattice/python:regularizers",
- "//tensorflow_lattice/python:tools",
- "//tensorflow_lattice/python/estimators:base",
- "//tensorflow_lattice/python/estimators:calibrated",
- "//tensorflow_lattice/python/estimators:calibrated_etl",
- "//tensorflow_lattice/python/estimators:calibrated_lattice",
- "//tensorflow_lattice/python/estimators:calibrated_linear",
- "//tensorflow_lattice/python/estimators:calibrated_rtl",
- "//tensorflow_lattice/python/estimators:hparams",
- "//tensorflow_lattice/python/estimators:separately_calibrated_rtl",
+ srcs = [
+ "__init__.py",
+ "layers/__init__.py",
],
-)
-
-# Depend on this if you have a C++ library or binary that uses TensorFlow
-# lattice ops.
-cc_library(
- name = "tensorflow_lattice_cc",
+ srcs_version = "PY2AND3",
deps = [
- "//tensorflow_lattice/cc:lattice_ops",
- "//tensorflow_lattice/cc:pwl_calibration_ops",
+ "//tensorflow_lattice/python:categorical_calibration_layer",
+ "//tensorflow_lattice/python:categorical_calibration_lib",
+ "//tensorflow_lattice/python:configs",
+ "//tensorflow_lattice/python:estimators",
+ "//tensorflow_lattice/python:lattice_layer",
+ "//tensorflow_lattice/python:lattice_lib",
+ "//tensorflow_lattice/python:linear_layer",
+ "//tensorflow_lattice/python:linear_lib",
+ "//tensorflow_lattice/python:model_info",
+ "//tensorflow_lattice/python:parallel_combination_layer",
+ "//tensorflow_lattice/python:pwl_calibration_layer",
+ "//tensorflow_lattice/python:pwl_calibration_lib",
+ "//tensorflow_lattice/python:test_utils",
+ "//tensorflow_lattice/python:visualization",
],
- alwayslink = 1,
)
diff --git a/tensorflow_lattice/__init__.py b/tensorflow_lattice/__init__.py
index 03fcc52..2aeca52 100644
--- a/tensorflow_lattice/__init__.py
+++ b/tensorflow_lattice/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2017 The TensorFlow Lattice Authors.
+# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,57 +11,27 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-# ==============================================================================
-"""Lattice modeling.
-
-This package provides functions and classes for lattice modeling.
-
-See full description in `README.md` file.
+"""Tensorflow Lattice Library.
- use them.
+This package provides functions and classes for lattice modeling.
"""
-# pylint: disable=unused-import,wildcard-import, line-too-long
-
from __future__ import absolute_import
-# Dependency imports
-
-# Import all modules here, but only import functions and classes that are
-# more likely to be used directly by users.
-from tensorflow_lattice.python.estimators.calibrated import input_calibration_layer_from_hparams
-from tensorflow_lattice.python.estimators.calibrated_etl import calibrated_etl_classifier
-from tensorflow_lattice.python.estimators.calibrated_etl import calibrated_etl_regressor
-from tensorflow_lattice.python.estimators.calibrated_lattice import calibrated_lattice_classifier
-from tensorflow_lattice.python.estimators.calibrated_lattice import calibrated_lattice_regressor
-from tensorflow_lattice.python.estimators.calibrated_linear import calibrated_linear_classifier
-from tensorflow_lattice.python.estimators.calibrated_linear import calibrated_linear_regressor
-from tensorflow_lattice.python.estimators.calibrated_rtl import calibrated_rtl_classifier
-from tensorflow_lattice.python.estimators.calibrated_rtl import calibrated_rtl_regressor
-from tensorflow_lattice.python.estimators.hparams import CalibratedEtlHParams
-from tensorflow_lattice.python.estimators.hparams import CalibratedHParams
-from tensorflow_lattice.python.estimators.hparams import CalibratedLatticeHParams
-from tensorflow_lattice.python.estimators.hparams import CalibratedLinearHParams
-from tensorflow_lattice.python.estimators.hparams import CalibratedRtlHParams
-from tensorflow_lattice.python.estimators.hparams import PerFeatureHParams
-from tensorflow_lattice.python.estimators.separately_calibrated_rtl import separately_calibrated_rtl_classifier
-from tensorflow_lattice.python.estimators.separately_calibrated_rtl import separately_calibrated_rtl_regressor
-from tensorflow_lattice.python.lib.keypoints_initialization import load_keypoints_from_quantiles
-from tensorflow_lattice.python.lib.keypoints_initialization import save_quantiles_for_keypoints
-from tensorflow_lattice.python.lib.keypoints_initialization import save_quantiles_for_keypoints_once
-from tensorflow_lattice.python.lib.keypoints_initialization import uniform_keypoints_for_signal
-from tensorflow_lattice.python.lib.lattice_layers import ensemble_lattices_layer
-from tensorflow_lattice.python.lib.lattice_layers import lattice_layer
-from tensorflow_lattice.python.lib.lattice_layers import monotone_lattice
-from tensorflow_lattice.python.lib.monotone_linear_layers import monotone_linear_layer
-from tensorflow_lattice.python.lib.monotone_linear_layers import split_monotone_linear_layer
-from tensorflow_lattice.python.lib.pwl_calibration_layers import calibration_layer
-from tensorflow_lattice.python.lib.pwl_calibration_layers import input_calibration_layer
-from tensorflow_lattice.python.lib.regularizers import calibrator_regularization
-from tensorflow_lattice.python.lib.regularizers import lattice_regularization
-from tensorflow_lattice.python.lib.tools import DEFAULT_NAME
-from tensorflow_lattice.python.ops.gen_monotonic_projection import monotonic_projection
-from tensorflow_lattice.python.ops.gen_pwl_indexing_calibrator import pwl_indexing_calibrator
-from tensorflow_lattice.python.ops.lattice_ops import lattice
-# pylint: enable=unused-import,wildcard-import,line-too-long
+import tensorflow_lattice.layers
+
+from tensorflow_lattice.python import categorical_calibration_layer
+from tensorflow_lattice.python import categorical_calibration_lib
+from tensorflow_lattice.python import configs
+from tensorflow_lattice.python import estimators
+from tensorflow_lattice.python import lattice_layer
+from tensorflow_lattice.python import lattice_lib
+from tensorflow_lattice.python import linear_layer
+from tensorflow_lattice.python import linear_lib
+from tensorflow_lattice.python import model_info
+from tensorflow_lattice.python import parallel_combination_layer
+from tensorflow_lattice.python import pwl_calibration_layer
+from tensorflow_lattice.python import pwl_calibration_lib
+from tensorflow_lattice.python import test_utils
+from tensorflow_lattice.python import visualization
diff --git a/tensorflow_lattice/cc/BUILD b/tensorflow_lattice/cc/BUILD
deleted file mode 100644
index d656718..0000000
--- a/tensorflow_lattice/cc/BUILD
+++ /dev/null
@@ -1,204 +0,0 @@
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-licenses(["notice"]) # Apache 2.0 License
-
-package(
- default_visibility = [
- "//tensorflow_lattice:__subpackages__",
- ],
-)
-
-exports_files(["LICENSE"])
-
-load(
- "//tensorflow_lattice:tensorflow_lattice.bzl",
- "rpath_linkopts",
-)
-load(
- "@org_tensorflow//tensorflow:tensorflow.bzl",
- "tf_cc_test",
- "tf_custom_op_library",
- "tf_gen_op_libs",
-)
-
-tf_custom_op_library(
- name = "ops/_lattice_ops.so",
- srcs = [
- ":ops/lattice_interpolation_ops.cc",
- ":ops/monotone_lattice_ops.cc",
- ],
- linkopts = rpath_linkopts("ops/_lattice_ops.so"),
- deps = [
- "//tensorflow_lattice/cc/kernels:lattice_kernels",
- "//tensorflow_lattice/cc/lib:lattice_structure",
- ],
-)
-
-tf_custom_op_library(
- name = "ops/_pwl_calibration_ops.so",
- srcs = [
- ":ops/monotonic_projection_op.cc",
- ":ops/pwl_indexing_calibrator_ops.cc",
- ],
- linkopts = rpath_linkopts("ops/_pwl_calibration_ops.so"),
- deps = [
- "//tensorflow_lattice/cc/kernels:pwl_calibration_kernels",
- ],
-)
-
-cc_library(
- name = "lattice_ops",
- deps = [
- ":lattice_interpolation_ops_op_lib",
- ":monotone_lattice_ops_op_lib",
- ],
- alwayslink = 1,
-)
-
-cc_library(
- name = "pwl_calibration_ops",
- deps = [
- ":monotonic_projection_op_op_lib",
- ":pwl_indexing_calibrator_ops_op_lib",
- ],
- alwayslink = 1,
-)
-
-# Collection of operators.
-tf_gen_op_libs(
- op_lib_names = ["pwl_indexing_calibrator_ops"],
- deps = [
- "//tensorflow_lattice/cc/kernels:pwl_indexing_calibrator_kernels",
- "@org_tensorflow//tensorflow/core:lib",
- ],
-)
-
-tf_gen_op_libs(
- op_lib_names = ["monotonic_projection_op"],
- deps = [
- "//tensorflow_lattice/cc/kernels:monotonic_projection_kernel",
- "@org_tensorflow//tensorflow/core:lib",
- ],
-)
-
-tf_gen_op_libs(
- op_lib_names = ["lattice_interpolation_ops"],
- deps = [
- "//tensorflow_lattice/cc/kernels:hypercube_interpolation_kernels",
- "//tensorflow_lattice/cc/kernels:lattice_interpolation_base",
- "//tensorflow_lattice/cc/kernels:simplex_interpolation_kernels",
- "//tensorflow_lattice/cc/lib:lattice_structure",
- "@org_tensorflow//tensorflow/core:lib",
- ],
-)
-
-tf_gen_op_libs(
- op_lib_names = ["monotone_lattice_ops"],
- deps = [
- "//tensorflow_lattice/cc/kernels:monotone_lattice_kernels",
- "//tensorflow_lattice/cc/lib:lattice_structure",
- "@org_tensorflow//tensorflow/core:lib",
- ],
-)
-
-# C++ tests.
-cc_library(
- name = "test_main",
- testonly = 1,
- srcs = ["test_tools/test_main.cc"],
- deps = [
- "@org_tensorflow//tensorflow/core:test",
- ],
-)
-
-tf_cc_test(
- name = "pwl_indexing_calibrator_ops_test",
- size = "small",
- srcs = ["ops/pwl_indexing_calibrator_ops_test.cc"],
- linkopts = rpath_linkopts("pwl_indexing_calibrator_ops_test"),
- deps = [
- ":pwl_indexing_calibrator_ops_op_lib",
- ":test_main",
- "@org_tensorflow//tensorflow/core:framework",
- "@org_tensorflow//tensorflow/core:lib",
- "@org_tensorflow//tensorflow/core:test",
- "@org_tensorflow//tensorflow/core:testlib",
- "@org_tensorflow//tensorflow/core/kernels:ops_testutil",
- ],
-)
-
-cc_library(
- name = "hypercube_interpolation_ops_test_lib",
- testonly = 1,
- srcs = ["ops/hypercube_interpolation_ops_test_p.cc"],
- hdrs = ["ops/hypercube_interpolation_ops_test.h"],
- linkopts = rpath_linkopts("hypercube_interpolation_ops_test"),
- deps = [
- ":lattice_interpolation_ops_op_lib",
- ":test_main",
- "@org_tensorflow//tensorflow/core:core_cpu",
- "@org_tensorflow//tensorflow/core:framework",
- "@org_tensorflow//tensorflow/core:lib",
- "@org_tensorflow//tensorflow/core:protos_all_cc",
- "@org_tensorflow//tensorflow/core:test",
- "@org_tensorflow//tensorflow/core:testlib",
- "@org_tensorflow//tensorflow/core/kernels:ops_testutil",
- "@org_tensorflow//tensorflow/core/kernels:ops_util_hdrs",
- ],
-)
-
-tf_cc_test(
- name = "hypercube_interpolation_ops_test",
- size = "small",
- srcs = ["ops/hypercube_interpolation_ops_test.cc"],
- deps = [
- ":hypercube_interpolation_ops_test_lib",
- "@org_tensorflow//tensorflow/core:framework",
- "@org_tensorflow//tensorflow/core:test",
- "@org_tensorflow//tensorflow/core:testlib",
- ],
-)
-
-tf_cc_test(
- name = "simplex_interpolation_ops_test",
- size = "small",
- srcs = ["ops/simplex_interpolation_ops_test.cc"],
- linkopts = rpath_linkopts("simplex_interpolation_ops_test"),
- deps = [
- ":lattice_interpolation_ops_op_lib",
- ":test_main",
- "@org_tensorflow//tensorflow/core:framework",
- "@org_tensorflow//tensorflow/core:lib",
- "@org_tensorflow//tensorflow/core:test",
- "@org_tensorflow//tensorflow/core:testlib",
- "@org_tensorflow//tensorflow/core/kernels:ops_testutil",
- ],
-)
-
-tf_cc_test(
- name = "monotonic_projection_op_test",
- size = "small",
- srcs = ["ops/monotonic_projection_op_test.cc"],
- linkopts = rpath_linkopts("monotonic_projection_op_test"),
- deps = [
- ":monotonic_projection_op_op_lib",
- ":test_main",
- "@org_tensorflow//tensorflow/core:framework",
- "@org_tensorflow//tensorflow/core:lib",
- "@org_tensorflow//tensorflow/core:test",
- "@org_tensorflow//tensorflow/core:testlib",
- "@org_tensorflow//tensorflow/core/kernels:ops_testutil",
- ],
-)
diff --git a/tensorflow_lattice/cc/kernels/BUILD b/tensorflow_lattice/cc/kernels/BUILD
deleted file mode 100644
index ca9b499..0000000
--- a/tensorflow_lattice/cc/kernels/BUILD
+++ /dev/null
@@ -1,166 +0,0 @@
-# Copyright 2017 The TensorFlow Lattice Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-licenses(["notice"]) # Apache 2.0
-
-package(
- default_visibility = [
- "//tensorflow_lattice:__subpackages__",
- ],
-)
-
-load(
- "//tensorflow_lattice:tensorflow_lattice.bzl",
- "rpath_linkopts",
-)
-load("@org_tensorflow//tensorflow:tensorflow.bzl", "tf_kernel_library")
-load("@org_tensorflow//tensorflow:tensorflow.bzl", "tf_cc_test")
-
-# Piecewise-linear calibration kernels
-cc_library(
- name = "pwl_calibration_kernels",
- deps = [
- ":monotonic_projection_kernel",
- ":pwl_indexing_calibrator_kernels",
- ],
-)
-
-tf_kernel_library(
- name = "pwl_indexing_calibrator_kernels",
- srcs = ["pwl_indexing_calibrator_kernels.cc"],
- deps = [
- "@org_tensorflow//tensorflow/core:framework_headers_lib",
- "@org_tensorflow//tensorflow/core:framework_lite",
- "@protobuf_archive//:protobuf",
- ],
-)
-
-tf_kernel_library(
- name = "monotonic_projection_kernel",
- srcs = ["monotonic_projection_kernel.cc"],
- deps = [
- ":monotonic_projections",
- "@org_tensorflow//tensorflow/core:framework_headers_lib",
- "@protobuf_archive//:protobuf",
- ],
-)
-
-# Lattice interpolation kernels
-cc_library(
- name = "lattice_kernels",
- deps = [
- ":hypercube_interpolation_kernels",
- ":monotone_lattice_kernels",
- ":simplex_interpolation_kernels",
- ],
-)
-
-cc_library(
- name = "lattice_interpolation_base",
- srcs = ["lattice_interpolation_base.cc"],
- hdrs = ["lattice_interpolation_base.h"],
- deps = [
- "//tensorflow_lattice/cc/lib:lattice_structure",
- "@org_tensorflow//tensorflow/core:framework_headers_lib",
- ],
-)
-
-tf_kernel_library(
- name = "hypercube_interpolation_kernels",
- srcs = ["hypercube_interpolation_kernels.cc"],
- deps = [
- ":lattice_interpolation_base",
- "//tensorflow_lattice/cc/lib:lattice_structure",
- "@org_tensorflow//tensorflow/core:framework_headers_lib",
- ],
- alwayslink = 1,
-)
-
-tf_kernel_library(
- name = "simplex_interpolation_kernels",
- srcs = ["simplex_interpolation_kernels.cc"],
- deps = [
- ":lattice_interpolation_base",
- "//tensorflow_lattice/cc/lib:lattice_structure",
- "@org_tensorflow//tensorflow/core:framework_headers_lib",
- ],
- alwayslink = 1,
-)
-
-# Monotonic projections.
-cc_library(
- name = "monotonic_projections",
- hdrs = ["monotonic_projections.h"],
- deps = [
- "@org_tensorflow//tensorflow/core:framework_headers_lib",
- "@org_tensorflow//tensorflow/core:framework_lite",
- ],
-)
-
-cc_library(
- name = "lattice_raw_iterator",
- srcs = ["lattice_raw_iterator.cc"],
- hdrs = ["lattice_raw_iterator.h"],
- deps = ["//tensorflow_lattice/cc/lib:lattice_structure"],
-)
-
-tf_cc_test(
- name = "lattice_raw_iterator_test",
- srcs = ["lattice_raw_iterator_test.cc"],
- linkopts = rpath_linkopts("lattice_raw_iterator_test"),
- deps = [
- ":lattice_raw_iterator",
- "//tensorflow_lattice/cc:test_main",
- "//tensorflow_lattice/cc/lib:lattice_structure",
- "@org_tensorflow//tensorflow/core:lib",
- "@org_tensorflow//tensorflow/core:test",
- ],
-)
-
-cc_library(
- name = "monotonic_lattice_projections",
- hdrs = ["monotonic_lattice_projections.h"],
- deps = [
- ":lattice_raw_iterator",
- ":monotonic_projections",
- "//tensorflow_lattice/cc/lib:lattice_structure",
- "@org_tensorflow//tensorflow/core:framework_headers_lib",
- ],
-)
-
-tf_cc_test(
- name = "monotonic_lattice_projections_test",
- srcs = ["monotonic_lattice_projections_test.cc"],
- linkopts = rpath_linkopts("monotonic_lattice_projections_test"),
- deps = [
- ":monotonic_lattice_projections",
- "//tensorflow_lattice/cc:test_main",
- "//tensorflow_lattice/cc/lib:lattice_structure",
- "@org_tensorflow//tensorflow/core:lib",
- "@org_tensorflow//tensorflow/core:test",
- ],
-)
-
-# Monotone lattice kernels.
-tf_kernel_library(
- name = "monotone_lattice_kernels",
- srcs = ["monotone_lattice_kernels.cc"],
- deps = [
- ":lattice_interpolation_base",
- ":monotonic_lattice_projections",
- "//tensorflow_lattice/cc/lib:lattice_structure",
- "@org_tensorflow//tensorflow/core:framework_headers_lib",
- ],
- alwayslink = 1,
-)
diff --git a/tensorflow_lattice/cc/kernels/hypercube_interpolation_kernels.cc b/tensorflow_lattice/cc/kernels/hypercube_interpolation_kernels.cc
deleted file mode 100644
index 7de1672..0000000
--- a/tensorflow_lattice/cc/kernels/hypercube_interpolation_kernels.cc
+++ /dev/null
@@ -1,348 +0,0 @@
-/* Copyright 2017 The TensorFlow Lattice Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include
-#include
-
-#include "tensorflow_lattice/cc/kernels/lattice_interpolation_base.h"
-#include "tensorflow_lattice/cc/lib/lattice_structure.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/platform/logging.h"
-
-namespace tensorflow {
-namespace lattice {
-
-// HypercubeInterpolationOpKernel returns interpolation weights.
-template
-class HypercubeInterpolationOpKernel
- : public LatticeInterpolationOpBase {
- public:
- explicit HypercubeInterpolationOpKernel(OpKernelConstruction* context)
- : LatticeInterpolationOpBase(context) {
-
- constexpr int64 kBaseCost = 20;
- constexpr int64 kCostPerCellVertex = 20;
- constexpr int64 kWeightInitializationCost = 1;
- this->SetCostPerExample(
- kCostPerCellVertex * this->GetLatticeStructure().NumVerticesPerCell() +
- kWeightInitializationCost * this->GetLatticeStructure().NumVertices() +
- kBaseCost);
- }
-
- private:
- InterpolationWeights ComputeInterpolationWeights(
- const LatticeStructure& lattice_structure,
- typename TTypes::UnalignedConstFlat input_vector) const final;
-
- TF_DISALLOW_COPY_AND_ASSIGN(HypercubeInterpolationOpKernel);
-};
-
-// HypercubeGradientOpKernel returns gradient with respect to the
-// input.
-template
-class HypercubeGradientOpKernel : public LatticeGradientOpBase {
- public:
- explicit HypercubeGradientOpKernel(OpKernelConstruction* context)
- : LatticeGradientOpBase(context) {
-
- constexpr int64 kBaseCost = 20;
- constexpr int64 kCostPerCellVertex = 20;
- this->SetCostPerExample(
- kCostPerCellVertex * this->GetLatticeStructure().Dimension() *
- this->GetLatticeStructure().NumVerticesPerCell() +
- kBaseCost);
- }
-
- private:
- std::vector ComputeGradWrtInput(
- const LatticeStructure& lattice_structure,
- typename TTypes::UnalignedConstFlat input_vector,
- typename TTypes::UnalignedConstFlat weight_vector,
- typename TTypes::UnalignedConstFlat grad_wrt_weight_vector)
- const final;
-
- TF_DISALLOW_COPY_AND_ASSIGN(HypercubeGradientOpKernel);
-};
-
-// Produces linear interpolation weights for an input that is in the unit
-// hypercube (the residual), as well as the corresponding indices in the lattice
-// (based on the bottom_corner). Both the weights and the indices are computed
-// during the same loop for efficiency, but we'll explain their computations
-// separately. Also returns the residual vector from the bottom corner of the
-// hypercube cell. See http://jmlr.org/papers/v17/15-243.html for more details.
-//
-// Calculating the linear interpolation weights
-// --------------------------------------------
-// The linear interpolation weights on each vertex are the volumes of the
-// hyperrectangles formed by partitioning the unit hypercube at the input.
-// Example: 2D case - Draw a unit square. Draw an input x in the square.
-// Draw horizontal and vertical lines through x. That forms 4 boxes - the
-// volume of these boxes are the weights. Note the boxes are a partition of
-// the unit square, so the sum of the areas (volumes) of the boxes sums to 1.
-// The linear interpolation weight on each vertex is the volume of the box in
-// the opposite corner (so that if you move x close to one corner, the weight
-// on that corner grows). Mathematically in the 2D case (and generalizes
-// directly to higher D) the weights are:
-// weight([0, 0]) = (1 - input[0]) * (1 - input[1])
-// weight([0, 1]) = (1 - input[0]) * input[1]
-// weight([1, 0]) = input[0] * (1 - input[1])
-// weight([1, 1]) = input[0] * input[1]
-//
-// Computing each of the 2^D weights directly using above formula would take
-// O(2^D * D) operations. Instead we take advantage of the many repeated
-// calculations to reduce this to a O(2^D) computation as follows:
-// Let's start by initializing weight to 1 for every vertex. Lets consider
-// current vertex. Suppose its bit representation is "0110". For every "0" we
-// should multiply its weight on (1 - input[i]), where i is a sequence number
-// of correspondent bit. And for each "1" we should multiply its weight on
-// input[i].
-// Let us iterate through all vertices in dfs (lexicographical) order. Let
-// current_highest_dimension be a sequence number of highest bit in binary
-// representation of current vertex. At this moment we multiplied
-// correspondent weights for all dimensions below current_highest_dimension.
-// Now, let us update current_highest_dimension.
-// Example:
-// If "ii" is iterating on "??x010" (the location of memory where finally the
-// weight for "00_1_010" will be stored), then we set the value for
-//
-// // Resetting bit x of ??x010.
-// earlier_ii = ii ^ (1 << current_highest_dimension)
-// // Now ii represents ?x1010.
-// weight[ii] = weight[earlier_ii] * input[current_highest_dimension]
-// // earlier_ii represents ?x0010.
-// weight[earlier_ii] *= (1 - input[current_highest_dimension])
-//
-// Example for 2x2 case:
-// weight[0] is weight on [0,0]
-// weight[1] is weight on [1,0]
-// weight[2] is weight on [0,1]
-// weight[3] is weight on [1,1]
-// Initialization: weight[0] = 1, no other weight set.
-// Loop: ii = 1. current_highest_dimension = 0
-// weight[1] = weight[0] * input[0];
-// weight[0] = weight[0] * (1 - input[0])
-// ii = 2. current_highest_dimension = 1. (highest bit of ii got index 1 at
-// this step, so update current_highest_dimension to reflect this)
-// weight[2] = weight[0] * input[1];
-// weight[0] = weight[0] * (1 - input[1])
-// ii = 3. current_highest_dimension = 1.
-// weight[3] = weight[1] * input[1];
-// weight[1] = weight[1] * (1 - input[1])
-//
-// Calculating the corresponding indices. Notice if the lattice sizes are larger
-// than 2, the indices of the wieghts will be adjusted according to the
-// LatticeStructure.strides.
-// -------------------------------------
-// The lattice index for the iith vertex in the cell is the same as the index
-// we computed for an earlier neighbor vertex, but offset by
-// lattice_strides[(dimensions - 1) - current_highest_dimension].
-// Example:
-// Suppose we have a 2x2 lattice. We should output vertices in the order:
-// [0,0], [1,0], [0,1], [1,1].
-// Bottom corner is [0,0], so vertices[0] = 0 already set.
-// let ii = 1. It corresponds to vertex [0,1]. current_highest_dimension = 0.
-// lattice index of vertices[1] is different from lattice index of
-// vertices[0] in dimension current_highest_dimension = 0 (counting from the
-// end).
-// vertices[1] = vertices[0] + lattice_strides[0] = [1, 0];
-// let ii = 2, it corresponds to [1,0]. current_highest_dimension becomes 1.
-// vertices[2] = vertices[0] + lattice_strides[1] = [0, 1];
-// vertices[3] = vertices[1] + lattice_strides[1] = [1, 1];
-//
-
-template
-InterpolationWeights
-HypercubeInterpolationOpKernel::ComputeInterpolationWeights(
- const LatticeStructure& lattice_structure,
- typename TTypes::UnalignedConstFlat input) const {
- const BottomCornerIndexAndResidual bottom_corner_index_and_residual =
- lattice_structure.GetBottomCornerIndexAndResidual(input);
- const std::vector& residual =
- bottom_corner_index_and_residual.residual;
- const int64 num_vertices_per_cell = lattice_structure.NumVerticesPerCell();
- // interpolation weight contains upto num_vertices_per_cell non-zero elements.
- InterpolationWeights interpolation_weights;
- std::vector& index = interpolation_weights.indices;
- std::vector& weight = interpolation_weights.weights;
-
- index.resize(num_vertices_per_cell);
- weight.resize(num_vertices_per_cell);
- index[0] = bottom_corner_index_and_residual.bottom_corner_index;
- weight[0] = 1.0;
-
- const int64 input_dim = lattice_structure.Dimension();
- const std::vector& strides = lattice_structure.Strides();
-
- int64 current_highest_dimension = 0;
- Dtype current_residual_value = residual[current_highest_dimension];
- for (int64 ii = 1; ii < num_vertices_per_cell; ++ii) {
- // Make sure that we're within the bounds of the unit hypercube.
- DCHECK_GE(current_residual_value, 0);
- DCHECK_LE(current_residual_value, 1);
- // Sanity check: current_highest_dimension has better respect the bounds.
- DCHECK_GE(current_highest_dimension, 0);
- DCHECK_LT(current_highest_dimension, input_dim);
-
- const int64 earlier_ii = ii ^ (1 << current_highest_dimension);
- index[ii] = index[earlier_ii] + strides[current_highest_dimension];
- weight[ii] = weight[earlier_ii] * current_residual_value;
- weight[earlier_ii] *= (1.0 - current_residual_value);
-
- if ((ii & (ii + 1)) == 0) {
- // If ii + 1 is power of 2, then current_highest_dimension has changed,
- // that means, that we are processing next dimension.
- ++current_highest_dimension;
- if (input_dim >= current_highest_dimension + 1) {
- current_residual_value = residual[current_highest_dimension];
- }
- }
- }
- return interpolation_weights;
-}
-
-// The goal of the gradient op is, given grad_wrt_weight:
-// (dy / dweight[0], dy / dweight[1], dy / dweight[2], dy / dweight[3]),
-// to compute the grad_wrt_input:
-// (dy / dx[0], ..., dy / dx[D-1]).
-//
-// We know that:
-// dy/dx[jj] = sum_{ii \in weights} dy/dweight[ii] * dweight[ii]/dx[jj]
-//
-// For dweight[ii]/dx[jj], we use the following observation.
-// For any 2 x ... x 2 lattices:
-// weight[ii] + weight[jj] == constant.
-// for all (ii, jj) pair such that ii ^ jj == 2 ** k and ii < jj. (This means
-// ii's kth vertex is 0, and jj's kth vertex is 1, and other vertices are same.)
-// Moreover, for such (ii, jj) pair, we have
-// dweight[ii] / dx[k] == -(weight[ii] + weight[jj])
-// dweight[jj] / dx[k] == (weight[ii] + weight[jj])
-//
-// To see this, let us consider 2 x 2 lattice case.
-//
-// Recall that
-// weight[0] = (1 - x[0]) * (1 - x[1])
-// weight[1] = x[0] * (1 - x[1])
-// weight[2] = (1 - x[0]) * x[1]
-// weight[3] = x[0] * x[1]
-//
-// Therefore,
-// dweight[0] / dx[0] = -(1 - x[1]) == -(weight[0] + weight[1])
-// dweight[1] / dx[0] = (1 - x[1]) == (weight[0] + weight[1])
-// dweight[2] / dx[0] = -x[1] == -(weight[2] + weight[3])
-// dweight[3] / dx[0] = x[1] == (weight[2] + weight[3]),
-// and
-// dweight[0] / dx[1] = -(1 - x[0]) == -(weight[0] + weight[2])
-// dweight[1] / dx[1] = -x[0] == -(weight[1] + weight[3])
-// dweight[2] / dx[1] = (1 - x[0]) == (weight[0] + weight[2])
-// dweight[3] / dx[1] = x[0] == (weight[1] + weight[3]).
-//
-// So the summation part marginalize the dependency of x[k], and the sign is
-// minus if the kth vertex is 0, and plus if the kth vertex is 1.
-// The following code computes the gradient using the (ii, jj) pair by
-// enumerating all indices whose kth vertex is 0.
-// In order to support the multi-cell lattice, the code constructs a list
-// (nnz_weight below) that maps the indices in the 2 x .... x 2 cell holding x
-// into indices in the multi-cell.
-//
-// Including this construction, the overall complexity is
-// O((input_dim + 2) * 2 ** (input_dim - 1)).
-//
-// Also when x[jj] < 0 or x[jj] > lattice_size[jj], the input is out of bound.
-// So the change in the input should not change the output, therefore the
-// gradient should be zero.
-//
-
-template
-std::vector HypercubeGradientOpKernel::ComputeGradWrtInput(
- const LatticeStructure& lattice_structure,
- typename TTypes::UnalignedConstFlat input,
- typename TTypes::UnalignedConstFlat weight,
- typename TTypes::UnalignedConstFlat grad_wrt_weight) const {
- const BottomCornerIndexAndResidual bottom_corner_index_and_residual =
- lattice_structure.GetBottomCornerIndexAndResidual(input);
- const int64 input_dim = lattice_structure.Dimension();
- std::vector grad_wrt_input(input_dim, 0.0);
-
- // There are at most 2 ** n number of non-zero elements in weight.
- // nnz_weight_index keeps the index of non-zero element in the weight.
- // The following loop enumerats all vertices in cell in the following order.
- // [0, 0, ..., 0], [1, 0, ...,0], [0, 1, ..., 0], ..., [1, 1, ..., 1].
- std::vector nnz_weight_index(lattice_structure.NumVerticesPerCell());
-
- int64 current_dim = 0;
- int64 current_bit = 1; // Always 1 << current_dim;
- nnz_weight_index[0] = bottom_corner_index_and_residual.bottom_corner_index;
- const std::vector& strides = lattice_structure.Strides();
- for (int64 ii = 1; ii < nnz_weight_index.size(); ++ii) {
- if ((ii & current_bit) == 0) {
- ++current_dim;
- current_bit <<= 1;
- }
- // ii - current_bit is the base.
- // ii is the current one, which is always an upper layer in the current
- // dimension.
- nnz_weight_index[ii] =
- nnz_weight_index[ii - current_bit] + strides[current_dim];
- }
-
- // Compute the gradient for each input.
- for (int64 ii = 0; ii < input_dim; ++ii) {
- // If out_of_bound, gradient is 0.
- if (bottom_corner_index_and_residual.out_of_bound[ii]) {
- continue;
- }
- // Only process the bottom faces.
- int64 bit = 1 << ii;
- int64 stride = strides[ii];
- Dtype grad_ii = 0.0;
- for (int64 index = 0; index < lattice_structure.NumVerticesPerCell();
- ++index) {
- // Upper face. Skip this index.
- if (index & bit) {
- continue;
- }
- // Bottom face.
- int64 lower_index = nnz_weight_index[index];
- int64 upper_index = lower_index + stride;
- grad_ii += (weight(lower_index) + weight(upper_index)) *
- (grad_wrt_weight(upper_index) - grad_wrt_weight(lower_index));
- }
- grad_wrt_input[ii] = grad_ii;
- }
-
- return grad_wrt_input;
-}
-// Register kernels for float and double.
-REGISTER_KERNEL_BUILDER(Name("HypercubeInterpolation")
- .Device(DEVICE_CPU)
- .TypeConstraint("Dtype"),
- HypercubeInterpolationOpKernel);
-
-REGISTER_KERNEL_BUILDER(Name("HypercubeInterpolation")
- .Device(DEVICE_CPU)
- .TypeConstraint("Dtype"),
- HypercubeInterpolationOpKernel);
-
-REGISTER_KERNEL_BUILDER(
- Name("HypercubeGradient").Device(DEVICE_CPU).TypeConstraint("Dtype"),
- HypercubeGradientOpKernel);
-
-REGISTER_KERNEL_BUILDER(Name("HypercubeGradient")
- .Device(DEVICE_CPU)
- .TypeConstraint("Dtype"),
- HypercubeGradientOpKernel);
-
-} // namespace lattice
-} // namespace tensorflow
diff --git a/tensorflow_lattice/cc/kernels/lattice_interpolation_base.cc b/tensorflow_lattice/cc/kernels/lattice_interpolation_base.cc
deleted file mode 100644
index aceedb9..0000000
--- a/tensorflow_lattice/cc/kernels/lattice_interpolation_base.cc
+++ /dev/null
@@ -1,54 +0,0 @@
-/* Copyright 2017 The TensorFlow Lattice Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow_lattice/cc/kernels/lattice_interpolation_base.h"
-
-#include
-
-#include "tensorflow_lattice/cc/lib/lattice_structure.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/lib/strings/str_util.h"
-
-namespace tensorflow {
-namespace lattice {
-
-using errors::InvalidArgument;
-using str_util::Join;
-
-LatticeOpBase::LatticeOpBase(OpKernelConstruction* context)
- : OpKernel(context), cost_per_example_(1.0) {
- std::vector lattice_sizes;
- OP_REQUIRES_OK(context, context->GetAttr("lattice_sizes", &lattice_sizes));
- OP_REQUIRES(context, LatticeStructure::IsValidLatticeSizes(lattice_sizes),
- InvalidArgument(Join(lattice_sizes, ","),
- " is not a valid lattice size"));
- lattice_structure_ =
- std::unique_ptr(new LatticeStructure(lattice_sizes));
-}
-
-void LatticeOpBase::CheckShape(OpKernelContext* context, const Tensor& tensor,
- const std::vector& expected_shape) const {
- OP_REQUIRES(context, tensor.dims() == expected_shape.size(),
- InvalidArgument("expect rank ", expected_shape.size(), "but got ",
- tensor.DebugString()));
-
- for (int ii = 0; ii < expected_shape.size(); ++ii) {
- OP_REQUIRES(context, tensor.dim_size(ii) == expected_shape[ii],
- InvalidArgument("expect ", ii, "-dim: ", expected_shape[ii],
- "but got ", tensor.DebugString()));
- }
-}
-
-} // namespace lattice
-} // namespace tensorflow
diff --git a/tensorflow_lattice/cc/kernels/lattice_interpolation_base.h b/tensorflow_lattice/cc/kernels/lattice_interpolation_base.h
deleted file mode 100644
index 19456bb..0000000
--- a/tensorflow_lattice/cc/kernels/lattice_interpolation_base.h
+++ /dev/null
@@ -1,233 +0,0 @@
-/* Copyright 2017 The TensorFlow Lattice Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// Lattice interpolation base class.
-#ifndef TENSORFLOW_LATTICE_CC_KERNELS_LATTICE_INTERPOLATION_BASE_H_
-#define TENSORFLOW_LATTICE_CC_KERNELS_LATTICE_INTERPOLATION_BASE_H_
-
-#include
-#include
-#include
-
-#include "tensorflow_lattice/cc/lib/lattice_structure.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/util/work_sharder.h"
-
-namespace tensorflow {
-namespace lattice {
-
-template
-struct InterpolationWeights {
- std::vector indices;
- std::vector weights;
-};
-
-// LatticeOpBase class contains common part of all lattice operators as lattice
-// structure initialization.
-class LatticeOpBase : public OpKernel {
- public:
- explicit LatticeOpBase(OpKernelConstruction* context);
-
- // Returns the lattice_structure.
- const LatticeStructure& GetLatticeStructure() const {
- return *lattice_structure_;
- }
-
- // Check whether the shape of tensor is same with expected_shape.
- void CheckShape(OpKernelContext* context, const Tensor& tensor,
- const std::vector& expected_shape) const;
-
- // Cost per example.
- const int64 CostPerExample() const { return cost_per_example_; }
- void SetCostPerExample(const int64 cost_per_example) {
- cost_per_example_ = cost_per_example;
- }
-
- private:
- std::unique_ptr lattice_structure_;
- int64 cost_per_example_;
-};
-
-// LatticeInterpolationOpBase is a base class for
-// HypercubeInterpolationOpKernel and SimplexInterpolationOpKernel.
-// The InterpolationWeights computation should be implemented in
-// ComputeInterpolationWeights method.
-template
-class LatticeInterpolationOpBase : public LatticeOpBase {
- public:
- explicit LatticeInterpolationOpBase(OpKernelConstruction* context)
- : LatticeOpBase(context) {}
-
- void Compute(OpKernelContext* context) override;
-
- protected:
- virtual InterpolationWeights ComputeInterpolationWeights(
- const LatticeStructure& lattice_structure,
- typename TTypes::UnalignedConstFlat input_vector) const = 0;
-
- private:
- // Apply InterpolationWeights to each slice of tensors.
- void BatchInterpolationWorker(const Tensor& input_tensor, const int start,
- const int limit,
- Tensor* interpolation_weights_tensor) const;
-};
-
-template
-void LatticeInterpolationOpBase::BatchInterpolationWorker(
- const Tensor& input_tensor, const int start, const int limit,
- Tensor* interpolation_weights_tensor) const {
- for (int ii = start; ii < limit; ++ii) {
- // Get iith input vector.
- const auto input_row_ii = input_tensor.Slice(ii, ii + 1);
-
- // Compute weight-index pairs.
- const InterpolationWeights interpolation_weights =
- ComputeInterpolationWeights(GetLatticeStructure(),
- input_row_ii.unaligned_flat());
-
- // Get iith interpolation weight vector (output).
- auto interpolation_weights_row_ii =
- interpolation_weights_tensor->Slice(ii, ii + 1).unaligned_flat();
-
- // Assign values to interpolation weight vector.
- interpolation_weights_row_ii.setZero();
- DCHECK_EQ(interpolation_weights.indices.size(),
- interpolation_weights.weights.size());
- for (int jj = 0; jj < interpolation_weights.indices.size(); ++jj) {
- interpolation_weights_row_ii(interpolation_weights.indices[jj]) =
- interpolation_weights.weights[jj];
- }
- }
-}
-
-template
-void LatticeInterpolationOpBase::Compute(OpKernelContext* context) {
- const LatticeStructure& lattice_structure = GetLatticeStructure();
- // Grab the input tensor.
- const Tensor& input_tensor = context->input(0);
- // Check the shapes.
- const int64 batch_dim = input_tensor.dim_size(0);
- const int64 input_dim = lattice_structure.Dimension();
- CheckShape(context, input_tensor, {batch_dim, input_dim});
-
- // Allocate interpolation_weights_tensor.
- Tensor* interpolation_weights_tensor = nullptr;
- OP_REQUIRES_OK(
- context,
- context->allocate_output(
- 0, TensorShape({batch_dim, lattice_structure.NumVertices()}),
- &interpolation_weights_tensor));
-
- auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
-
- // Launch threads.
- Shard(worker_threads.num_threads, worker_threads.workers, batch_dim,
- CostPerExample(), [&](int start, int limit) {
- BatchInterpolationWorker(input_tensor, start, limit,
- interpolation_weights_tensor);
- });
-}
-
-// LatticeGradientOpBase is a base class for HypercubeGradientOpKernel and
-// SimplexGradientOpKernel.
-// Computing Gradient with respect to input should be should be implemented in
-// ComputeGradWrtInput method.
-template
-class LatticeGradientOpBase : public LatticeOpBase {
- public:
- explicit LatticeGradientOpBase(OpKernelConstruction* context)
- : LatticeOpBase(context) {}
-
- void Compute(OpKernelContext* context) override;
-
- protected:
- virtual std::vector ComputeGradWrtInput(
- const LatticeStructure& lattice_structure,
- typename TTypes::UnalignedConstFlat input_vector,
- typename TTypes::UnalignedConstFlat weight_vector,
- typename TTypes::UnalignedConstFlat grad_wrt_weight_vector)
- const = 0;
-
- private:
- // Apply grad_wrt_input_fn_ to each slice of tensors.
- void BatchGradientWorker(const Tensor& input_tensor,
- const Tensor& weight_tensor,
- const Tensor& grad_wrt_weight_tensor,
- const int start, const int limit,
- Tensor* grad_wrt_input_tensor) const;
-};
-
-// BatchGradientWorker computes the gradient with respect to the input of each
-// row.
-template
-void LatticeGradientOpBase::BatchGradientWorker(
- const Tensor& input_tensor, const Tensor& weight_tensor,
- const Tensor& grad_wrt_weight_tensor, const int start, const int limit,
- Tensor* grad_wrt_input_tensor) const {
- auto grad_wrt_input_matrix = grad_wrt_input_tensor->matrix();
- for (int ii = start; ii < limit; ++ii) {
- const auto input_row_ii = input_tensor.Slice(ii, ii + 1);
- const auto weight_row_ii = weight_tensor.Slice(ii, ii + 1);
- const auto grad_wrt_weight_row_ii =
- grad_wrt_weight_tensor.Slice(ii, ii + 1);
-
- const std::vector grad_wrt_input = ComputeGradWrtInput(
- GetLatticeStructure(), input_row_ii.unaligned_flat(),
- weight_row_ii.unaligned_flat(),
- grad_wrt_weight_row_ii.unaligned_flat());
-
- for (int jj = 0; jj < grad_wrt_input.size(); ++jj) {
- grad_wrt_input_matrix(ii, jj) = grad_wrt_input[jj];
- }
- }
-}
-
-template
-void LatticeGradientOpBase::Compute(OpKernelContext* context) {
- const LatticeStructure& lattice_structure = this->GetLatticeStructure();
- const Tensor& input_tensor = context->input(0);
- const Tensor& weight_tensor = context->input(1);
- const Tensor& grad_wrt_weight_tensor = context->input(2);
- // Check the shapes.
- const int64 batch_dim = input_tensor.dim_size(0);
- const int64 input_dim = lattice_structure.Dimension();
- CheckShape(context, input_tensor, {batch_dim, input_dim});
- CheckShape(context, weight_tensor,
- {batch_dim, lattice_structure.NumVertices()});
- CheckShape(context, grad_wrt_weight_tensor,
- {batch_dim, lattice_structure.NumVertices()});
-
- // Dense implementation.
- Tensor* grad_wrt_input_tensor = nullptr;
- OP_REQUIRES_OK(
- context,
- context->allocate_output(0, TensorShape({batch_dim, input_dim}),
- &grad_wrt_input_tensor));
-
- auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
-
- // Launch threads.
- Shard(worker_threads.num_threads, worker_threads.workers, batch_dim,
- CostPerExample(), [&](int start, int limit) {
- BatchGradientWorker(input_tensor, weight_tensor,
- grad_wrt_weight_tensor, start, limit,
- grad_wrt_input_tensor);
- });
-}
-
-} // namespace lattice
-} // namespace tensorflow
-
-#endif // TENSORFLOW_LATTICE_CC_KERNELS_LATTICE_INTERPOLATION_BASE_H_
diff --git a/tensorflow_lattice/cc/kernels/lattice_raw_iterator.cc b/tensorflow_lattice/cc/kernels/lattice_raw_iterator.cc
deleted file mode 100644
index 101dfc1..0000000
--- a/tensorflow_lattice/cc/kernels/lattice_raw_iterator.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Copyright 2017 The TensorFlow Lattice Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow_lattice/cc/kernels/lattice_raw_iterator.h"
-
-#include
-
-#include "tensorflow_lattice/cc/lib/lattice_structure.h"
-
-namespace tensorflow {
-namespace lattice {
-
-void LatticeRawIterator::Next() {
- ++index_;
- for (int64 dim = 0; dim < lattice_sizes_.size(); ++dim) {
- ++vertex_[dim];
- if (vertex_[dim] == lattice_sizes_[dim]) {
- vertex_[dim] = 0;
- } else {
- break;
- }
- }
-}
-
-} // namespace lattice
-} // namespace tensorflow
diff --git a/tensorflow_lattice/cc/kernels/lattice_raw_iterator.h b/tensorflow_lattice/cc/kernels/lattice_raw_iterator.h
deleted file mode 100644
index 68e9fb9..0000000
--- a/tensorflow_lattice/cc/kernels/lattice_raw_iterator.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/* Copyright 2017 The TensorFlow Lattice Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// LatticeRawIterator iterates all vertices in a multi-cell lattice in the
-// column-major order. Note that this indexing (column-major order) should be
-// consistent with LatticeStructure.
-//
-// Iteration example:
-// for (LatticeRawIterator iter(lattice_structure) ; !iter.IsDone();
-// iter.Next()) {
-// const int64 global_index = iter.Index();
-// const int64 vertex_first_dim = iter.VertexDim(0);
-// const int64 vertex_second_dim = iter.VertexDim(1);
-// }
-
-#ifndef TENSORFLOW_LATTICE_CC_KERNELS_LATTICE_RAW_ITERATOR_H_
-#define TENSORFLOW_LATTICE_CC_KERNELS_LATTICE_RAW_ITERATOR_H_
-
-#include
-#include
-
-#include "tensorflow_lattice/cc/lib/lattice_structure.h"
-
-namespace tensorflow {
-namespace lattice {
-
-class LatticeRawIterator {
- public:
- explicit LatticeRawIterator(const LatticeStructure& lattice_structure)
- : lattice_sizes_(lattice_structure.LatticeSizes()),
- vertex_(lattice_structure.Dimension(), 0),
- index_(0),
- last_index_(lattice_structure.NumVertices()) {}
-
- // Forwards the iterator.
- void Next();
-
- bool IsDone() const { return index_ >= last_index_; }
- int64 Index() const { return index_; }
- const std::vector& Vertex() const { return vertex_; }
- int64 VertexDim(const int64 dim) const { return vertex_[dim]; }
-
- private:
- const std::vector lattice_sizes_;
- std::vector vertex_;
- int64 index_;
- const int64 last_index_;
-};
-
-} // namespace lattice
-} // namespace tensorflow
-#endif // TENSORFLOW_LATTICE_CC_KERNELS_LATTICE_RAW_ITERATOR_H_
diff --git a/tensorflow_lattice/cc/kernels/lattice_raw_iterator_test.cc b/tensorflow_lattice/cc/kernels/lattice_raw_iterator_test.cc
deleted file mode 100644
index 144825c..0000000
--- a/tensorflow_lattice/cc/kernels/lattice_raw_iterator_test.cc
+++ /dev/null
@@ -1,111 +0,0 @@
-/* Copyright 2017 The TensorFlow Lattice Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow_lattice/cc/kernels/lattice_raw_iterator.h"
-
-#include
-
-#include "tensorflow_lattice/cc/lib/lattice_structure.h"
-#include "tensorflow/core/lib/strings/str_util.h"
-#include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/platform/test.h"
-
-namespace tensorflow {
-namespace lattice {
-
-namespace {
-struct IndexVertexPair {
- int64 index;
- std::vector vertex;
-};
-} // namespace
-
-// The fixture for testing LatticeRawIteration.
-class LatticeRawIteratorTest : public ::testing::Test {
- protected:
- // Given the lattice sizes, iterate using RawIterator and check whether the
- // iterator visits all expected index_vertex_pairs.
- void CheckFullIteration(
- const std::vector& lattice_sizes,
- const std::vector& expected_index_vertex_pairs) {
- LatticeStructure lattice_structure(lattice_sizes);
-
- // Iterate and collect indices and vertices.
- std::vector visited_index_vertex_pairs;
- for (LatticeRawIterator iter(lattice_structure); !iter.IsDone();
- iter.Next()) {
- visited_index_vertex_pairs.push_back(
- IndexVertexPair{iter.Index(), iter.Vertex()});
- LOG(INFO) << "visited_index : " << iter.Index() << " visited_vertex: ["
- << str_util::Join(iter.Vertex(), ",") << "]";
- }
-
- // Check the result with the expected results.
- CompareIndexVertexPairs(expected_index_vertex_pairs,
- visited_index_vertex_pairs);
- }
-
- private:
- void CompareIndexVertexPairs(
- const std::vector& index_vertex_pairs1,
- const std::vector& index_vertex_pairs2) {
- ASSERT_EQ(index_vertex_pairs1.size(), index_vertex_pairs2.size());
- const int num_pairs = index_vertex_pairs1.size();
- std::vector visited(num_pairs, false);
- // n ** 2 comparsion.
- for (const auto& index_vertex_pair2 : index_vertex_pairs2) {
- for (int ii = 0; ii < num_pairs; ++ii) {
- if (index_vertex_pair2.index == index_vertex_pairs1[ii].index &&
- index_vertex_pair2.vertex == index_vertex_pairs1[ii].vertex) {
- visited[ii] = true;
- break;
- }
- }
- }
- // Now check that we visited all index_vertex_pair in index_vertex_pairs1.
- for (const bool is_visited : visited) {
- EXPECT_TRUE(is_visited);
- }
- }
-};
-
-TEST_F(LatticeRawIteratorTest, FullIterationWithTwoByThree) {
- CheckFullIteration(
- /*lattice_sizes=*/{2, 3}, /*expected_index_vertex_pairs=*/{{0, {0, 0}},
- {1, {1, 0}},
- {2, {0, 1}},
- {3, {1, 1}},
- {4, {0, 2}},
- {5, {1, 2}}});
-}
-
-TEST_F(LatticeRawIteratorTest, FullIterationWithThreeByTwoByTwo) {
- CheckFullIteration(
- /*lattice_sizes=*/{3, 2, 2},
- /*expected_index_vertex_pairs=*/{{0, {0, 0, 0}},
- {1, {1, 0, 0}},
- {2, {2, 0, 0}},
- {3, {0, 1, 0}},
- {4, {1, 1, 0}},
- {5, {2, 1, 0}},
- {6, {0, 0, 1}},
- {7, {1, 0, 1}},
- {8, {2, 0, 1}},
- {9, {0, 1, 1}},
- {10, {1, 1, 1}},
- {11, {2, 1, 1}}});
-}
-
-} // namespace lattice
-} // namespace tensorflow
diff --git a/tensorflow_lattice/cc/kernels/monotone_lattice_kernels.cc b/tensorflow_lattice/cc/kernels/monotone_lattice_kernels.cc
deleted file mode 100644
index 6ff06ea..0000000
--- a/tensorflow_lattice/cc/kernels/monotone_lattice_kernels.cc
+++ /dev/null
@@ -1,184 +0,0 @@
-/* Copyright 2017 The TensorFlow Lattice Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include
-#include
-
-#include "tensorflow_lattice/cc/kernels/lattice_interpolation_base.h"
-#include "tensorflow_lattice/cc/kernels/monotonic_lattice_projections.h"
-#include "tensorflow_lattice/cc/lib/lattice_structure.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/util/work_sharder.h"
-
-namespace tensorflow {
-namespace lattice {
-
-// MonotoneLatticeOp returns the projected lattice param vectors onto the
-// subspace that satisfies monotonicity constraints specified by is_monotone.
-// If is_monotone[k] == true, then kth input will have a non-decreasing
-// monotonicity constraint, and is_monotone[k] == false, then then kth input has
-// no monotonicity constraints.
-//
-// Lattice param tensor is expected to be a 2d tensor, [num_outputs,
-// num_parameters], where each row represents a parameter from multi-cell
-// lattice.
-template
-class MonotoneLatticeOp : public LatticeOpBase {
- public:
- static_assert(std::is_floating_point::value,
- "Dtype needs to be a floating point");
-
- explicit MonotoneLatticeOp(OpKernelConstruction* context);
- void Compute(OpKernelContext* context) final;
-
- TF_DISALLOW_COPY_AND_ASSIGN(MonotoneLatticeOp);
-
- private:
- void ProjectionWorker(const Tensor& lattice_params_tensor, int start,
- int limit, int num_parameters,
- Tensor* projection_tensor,
- OpKernelContext* context) const;
-
- std::unique_ptr> projector_;
-};
-
-template
-MonotoneLatticeOp::MonotoneLatticeOp(OpKernelConstruction* context)
- : LatticeOpBase(context) {
- std::vector is_monotone;
- float tolerance;
- int64 max_iter;
-
- OP_REQUIRES_OK(context, context->GetAttr("is_monotone", &is_monotone));
- OP_REQUIRES_OK(context, context->GetAttr("tolerance", &tolerance));
- OP_REQUIRES_OK(context, context->GetAttr("max_iter", &max_iter));
-
- const int64 lattice_dim = GetLatticeStructure().Dimension();
- OP_REQUIRES(context, (is_monotone.size() == lattice_dim),
- errors::InvalidArgument(
- "lattice dimension :", lattice_dim,
- " != ", "is_monotone dimension: ", is_monotone.size()));
-
- std::vector monotone_dims;
- for (int ii = 0; ii < lattice_dim; ++ii) {
- if (is_monotone[ii]) {
- monotone_dims.push_back(ii);
- }
- }
-
- projector_ = std::unique_ptr>(
- new MonotoneLatticeProjector(GetLatticeStructure(), monotone_dims,
- tolerance, max_iter));
-
-
- constexpr int64 kInitCost = 20;
- constexpr int64 kBaseCost = 20;
- constexpr int64 kConstraintCost = 20;
- // For initilaization: constant0 * GetLatticeStructure().NumVertices().
- // Each iteration in ADMM:
- // 1. Projection for each constraint: constant1 * NumVertices().
- // 2. Center variable update: constant2 * NumVertices()
- // 3. Dual variable update for each constraint: constant3 *
- // NumVertices().
- // Therefore, the total cost of each iteration is
- // ((constant1 + constant3) * number of monotone dimensions + constant2) *
- // NumVertices().
- // The number of iteration is bounded by min(max_iter, O(||true_projection -
- // initial_point||_2/epsilon)). But since the latter is hard to obtain, we use
- // max_iter as an upper bound.
- // So the total cost is given by
- //
- // ((kConstraintCost * monotone_dims.size() + kBaseCost) * max_iter +
- // kInitCost) * GetLatticeStructure().NumVertices()
- const int64 cost_per_example =
- ((kConstraintCost * monotone_dims.size() + kBaseCost) * max_iter +
- kInitCost) *
- GetLatticeStructure().NumVertices();
- SetCostPerExample(cost_per_example);
-}
-
-template
-void MonotoneLatticeOp::Compute(OpKernelContext* context) {
- // Grab the param tensor. Expect [num_ouputs, num_parameters] tensor.
- const Tensor& lattice_params_tensor = context->input(0);
-
- OP_REQUIRES(context, lattice_params_tensor.dims() == 2,
- errors::InvalidArgument("expected a 2d tensor, got ",
- lattice_params_tensor.dims()));
- OP_REQUIRES(
- context,
- lattice_params_tensor.dim_size(1) == GetLatticeStructure().NumVertices(),
- errors::InvalidArgument(
- "expected parameter dimension: ", GetLatticeStructure().NumVertices(),
- "got: ", lattice_params_tensor.dim_size(1)));
- const int64 num_outputs = lattice_params_tensor.dim_size(0);
- const int64 num_parameters = lattice_params_tensor.dim_size(1);
-
- Tensor* projection_tensor = nullptr;
- OP_REQUIRES_OK(context, context->allocate_output(
- 0, TensorShape({num_outputs, num_parameters}),
- &projection_tensor));
-
- auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
-
- // A worker that projects lattice_params_tensor[start : start + limit - 1, :]
- // and saves the result to from the
- // projection_tensor[start : start + limit - 1, :].
- // This lambda captures everything including "this" to use ProjectionWorker
- // method and all of captured states' lifetime is longer than Shard operation.
- auto worker = [&](int start, int limit) {
- ProjectionWorker(lattice_params_tensor, start, limit, num_parameters,
- projection_tensor, context);
- };
- // Launch threads.
- Shard(worker_threads.num_threads, worker_threads.workers, num_outputs,
- CostPerExample(), worker);
-}
-
-template
-void MonotoneLatticeOp::ProjectionWorker(
- const Tensor& lattice_params_tensor, const int start, const int limit,
- const int num_parameters, Tensor* projection_tensor,
- OpKernelContext* context) const {
- auto lattice_params_matrix = lattice_params_tensor.matrix();
- auto projection_matrix = projection_tensor->matrix();
- for (int row = start; row < limit; ++row) {
- // Computing the projection per each row.
- std::vector lattice_params_vec(num_parameters);
- std::vector projected_lattice_params_vec(num_parameters, 0.0);
-
- // Fetching the lattice parameter.
- for (int ii = 0; ii < num_parameters; ++ii) {
- lattice_params_vec[ii] = lattice_params_matrix(row, ii);
- }
- OP_REQUIRES_OK(context, projector_->Project(lattice_params_vec,
- &projected_lattice_params_vec));
- // Fill-in projected params.
- for (int ii = 0; ii < num_parameters; ++ii) {
- projection_matrix(row, ii) = projected_lattice_params_vec[ii];
- }
- }
-}
-
-// Register kernels for float and double.
-REGISTER_KERNEL_BUILDER(
- Name("MonotoneLattice").Device(DEVICE_CPU).TypeConstraint("Dtype"),
- MonotoneLatticeOp);
-REGISTER_KERNEL_BUILDER(
- Name("MonotoneLattice").Device(DEVICE_CPU).TypeConstraint("Dtype"),
- MonotoneLatticeOp);
-
-} // namespace lattice
-} // namespace tensorflow
diff --git a/tensorflow_lattice/cc/kernels/monotonic_lattice_projections.h b/tensorflow_lattice/cc/kernels/monotonic_lattice_projections.h
deleted file mode 100644
index 767272e..0000000
--- a/tensorflow_lattice/cc/kernels/monotonic_lattice_projections.h
+++ /dev/null
@@ -1,326 +0,0 @@
-/* Copyright 2017 The TensorFlow Lattice Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// Project lattice parameter vector onto monotonicity constraints.
-#ifndef TENSORFLOW_LATTICE_CC_KERNELS_MONOTONIC_LATTICE_PROJECTIONS_H_
-#define TENSORFLOW_LATTICE_CC_KERNELS_MONOTONIC_LATTICE_PROJECTIONS_H_
-
-#include
-#include
-#include
-#include
-
-#include "tensorflow_lattice/cc/kernels/lattice_raw_iterator.h"
-#include "tensorflow_lattice/cc/kernels/monotonic_projections.h"
-#include "tensorflow_lattice/cc/lib/lattice_structure.h"
-
-#include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/core/platform/logging.h"
-
-namespace tensorflow {
-namespace lattice {
-
-// Monotone Lattice projector projects lattice parameter to monotonicity
-// constraints specified by monotone_dimensions.
-// monotone_dimensions contains a index of (increasing) monotonic dimension.
-// For example, if we want to impose the monotonicity constraint in the 0th and
-// 2th dimensions, then monotone_dimensions = {0, 2}.
-//
-// The implementation uses Alternating Direction Method of Multipliers (ADMM)
-// parallel projection. See Distributed Optimization and Statistical Learning
-// via the Alternating Direction Method of Multipliers
-// (http://web.stanford.edu/~boyd/papers/pdf/admm_distr_stats.pdf) Section
-// 5.1.2. Parallel projeciton and Chapter 7. for the theoritical background.
-//
-// Suppose we have K number of convex sets, C_1, ..., C_K, and we want to
-// project a variable x_0 in R^n to the intersection of C_1, ..., C_K.
-// Let x_1, ..., x_K in R^n, and d_1, ..., d_K in R^n.
-// The ADMM parallel projection works as follows.
-//
-// Step 0: Initialize x_center = x_0, and d_k = 0 for all k = 1, ..., K.
-// Step 1: x_k <- Projection of (d_k + x_center) onto C_i for all i = 1, ...,
-// K.
-// Step 2: x_center <- 0.5 * x_0 + 0.5 * 1/K * sum_k (x_k - d_k).
-// Step 3: d_k <- d_k + x_center - x_k.
-// Step 4: Go back to the Step 1 if sum_k ||x_center - x_k||_1 > eps.
-//
-// Step 1 generates x_k in C_k. However, x_k may not be in C_i where i \neq k.
-// However, the algorithm is guaranteed to converge, which implies d_k should
-// stop being updated after many iterations.
-// Therefore, x_center == x_k for all k eventually. Since x_center == x_1 == ...
-// == x_K, we can conclude that x_center is in the intersection of C_1, ...,
-// C_K.
-// Step 2 generates x_center that minimizes ||x_center - x_0||_2^2 + some
-// regularization terms. Upon convergence, regularization temrs are zero.
-// Therefore, x_center == the projeciton of x_0 onto the intersection of C_1,
-// ..., C_K, when the algorithm converges.
-//
-// In the following implementation, we set each C_k to be the set of
-// lattice_param_vec that satisfies one dimensional monotonicity constraint.
-// Assuming we have K number of monotone dimensions, the ADMM algorithm
-// perform the projection for a given lattice_param_vec as follows:
-// Step 0: Initialize center = lattice_param_vec and duals[k] =
-// std::vector(param_size, 0.0) for k = 0, ..., K - 1.
-// Step 1: params[k] <- Projection of (duals[k] + center) onto the kth 1D
-// monotonicity constraint. (Here + means an elementwise summation.) for k =
-// 0, ..., K - 1.
-// Step 2: center <- 0.5 * lattice_param_vec + 0.5 * 1/K * sum_k (params[k] -
-// duals[k])
-// Step 3: duals[k] += (center - params[k]) for k = 0, ..., K - 1.
-// Step 4: Repeat Step 1 until sum_k ||center - params[k]||_1 < epsilon, or
-// Step 1 was repeated more than max_iter times.
-template
-class MonotoneLatticeProjector {
- public:
- static_assert(std::is_floating_point::value,
- "Dtype needs to be a floating point");
-
- explicit MonotoneLatticeProjector(const LatticeStructure& lattice_structure,
- const std::vector& monotone_dimensions,
- const Dtype epsilon = 1e-7,
- const int64 max_iter = 100000);
-
- // Apply ADMM projections, and save the result to the projected_param.
- Status Project(const std::vector& lattice_param_vec,
- std::vector* projected_lattice_param_vec) const;
-
- private:
- // This projector computes the projection of lattice parameter vector onto the
- // per dimension monotonicity constraints.
- //
- // For example, consider 3 x 3 lattice:
- //
- // 2---------5--------8
- // | | |
- // | | |
- // 1---------4--------7
- // | | |
- // | | |
- // 0---------3--------6
- //
- // For the 0th dimension, we have
- // weight[0] <= weight[3] <= weight[6]
- // weight[1] <= weight[4] <= weight[7]
- // weight[2] <= weight[5] <= weight[8].
- //
- // So PerDimensionProjector(lattice_structure, 0) will project the
- // lattice_param_vec onto the constraints of the given dimension.
- //
- // For the 1th dimension, we have
- // weight[0] <= weight[1] <= weight[2]
- // weight[3] <= weight[4] <= weight[5]
- // weight[6] <= weight[7] <= weight[8].
- //
- // So PerDimensionProjector(lattice_structure, 1) will project
- // lattice_param_vec onto the constraints of the given dimension.
- class PerDimensionProjector {
- public:
- explicit PerDimensionProjector(const LatticeStructure& lattice_structure,
- const int64 dimension);
-
- // Apply projection, and save the result to the lattice_param_vec.
- void Project(std::vector* lattice_param_vec) const;
-
- private:
- // Helper function that returns the base indices of a given LatticeStructure
- // and dimension.
- static std::vector BaseIndices(
- const LatticeStructure& lattice_structure, const int64 dimension);
-
- const int64 lattice_size_;
- const int64 stride_;
- const std::vector base_indices_;
- };
-
- const Dtype epsilon_;
- const int64 max_iter_;
- int64 param_size_;
- std::vector projectors_;
-};
-
-// Implementation of PerDimensionProjector's methods.
-template
-MonotoneLatticeProjector::PerDimensionProjector::PerDimensionProjector(
- const LatticeStructure& lattice_structure, const int64 dimension)
- : lattice_size_(lattice_structure.LatticeSize(dimension)),
- stride_(lattice_structure.Stride(dimension)),
- base_indices_(BaseIndices(lattice_structure, dimension)) {}
-
-template
-std::vector
-MonotoneLatticeProjector::PerDimensionProjector::BaseIndices(
- const LatticeStructure& lattice_structure, const int64 dimension) {
- std::vector base_indices;
-
- for (LatticeRawIterator iter(lattice_structure); !iter.IsDone();
- iter.Next()) {
- if (iter.VertexDim(dimension) == 0) {
- base_indices.push_back(iter.Index());
- }
- }
- return base_indices;
-}
-
-
-template
-void MonotoneLatticeProjector::PerDimensionProjector::Project(
- std::vector* lattice_param_vec_ptr) const {
- DCHECK(lattice_param_vec_ptr);
-
- std::vector& lattice_param_vec = *lattice_param_vec_ptr;
- for (const int64 base_index : base_indices_) {
- std::vector lattice_slice(lattice_size_);
- // Find the slice of lattice parameter vector.
- int64 current_index = base_index;
- for (Dtype& value : lattice_slice) {
- value = lattice_param_vec[current_index];
- current_index += stride_;
- }
-
- // Make a projection.
- std::vector projected_slice =
- VectorMonotonicProjection(lattice_slice, std::less_equal());
-
- // Fill in the result.
- current_index = base_index;
- for (const Dtype value : projected_slice) {
- lattice_param_vec[current_index] = value;
- current_index += stride_;
- }
- }
-}
-
-// Implementation of MonotoneLatticeProjector's methods.
-template
-MonotoneLatticeProjector::MonotoneLatticeProjector(
- const LatticeStructure& lattice_structure,
- const std::vector& monotone_dimensions, const Dtype epsilon,
- const int64 max_iter)
- : epsilon_(epsilon),
- max_iter_(max_iter),
- param_size_(lattice_structure.NumVertices()) {
- for (const int dim : monotone_dimensions) {
- projectors_.push_back(PerDimensionProjector(lattice_structure, dim));
- }
-}
-
-// Apply ADMM projections.
-template
-Status MonotoneLatticeProjector::Project(
- const std::vector& lattice_param_vec,
- std::vector* projected_lattice_param_vec) const {
- if (lattice_param_vec.size() != param_size_) {
- return errors::InvalidArgument("lattice_param_vec's size (",
- lattice_param_vec.size(),
- ") != param_size (", param_size_, ")");
- }
-
- if (!projected_lattice_param_vec) {
- return errors::InvalidArgument("projected_lattice_param_vec is nullptr");
- }
- if (projected_lattice_param_vec->size() != param_size_) {
- return errors::InvalidArgument("projected_lattice_param_vec's size (",
- projected_lattice_param_vec->size(),
- ") != param_size (", param_size_, ")");
- }
-
- // No projection at all. Make a deep copy, then return.
- if (projectors_.empty()) {
- *projected_lattice_param_vec = lattice_param_vec;
- return Status::OK();
- }
-
- // Only one projection. No need for running a complicated projection.
- if (projectors_.size() == 1) {
- // Make a deep copy, then project.
- *projected_lattice_param_vec = lattice_param_vec;
- projectors_[0].Project(projected_lattice_param_vec);
- return Status::OK();
- }
-
- // Initialize all variables.
- // 1. Center: This contains a reference to the projected lattice parameter
- // vector.
- // 2. Param_per_cluster.
- // 3. Deviation_per_cluster.
- std::vector& center = *projected_lattice_param_vec;
- const int param_size = lattice_param_vec.size();
- const int num_clusters = projectors_.size();
-
- // Initial point is a deep copy of lattice_param_vec.
- center = lattice_param_vec;
- std::vector> param_per_cluster(
- num_clusters, std::vector(param_size, 0.0));
- std::vector> duals(num_clusters,
- std::vector(param_size, 0.0));
-
- Dtype residual = std::numeric_limits::max();
- int64 iter = 0;
- const Dtype average_scale = 0.5 / static_cast(num_clusters);
-
-
- while (residual > epsilon_) {
- // Step 1. Update parameter in each cluster by applying projections.
- for (int ii = 0; ii < num_clusters; ++ii) {
- // Step 1-1. Update param_per_cluster[ii] == center + duals[ii].
- const std::vector& duals_ii = duals[ii];
- std::vector& param_ii = param_per_cluster[ii];
- for (int jj = 0; jj < param_size; ++jj) {
- param_ii[jj] = duals_ii[jj] + center[jj];
- }
- // Step 1-2. Project onto the monotonicity constraint.
- projectors_[ii].Project(¶m_ii);
- }
-
- // Step 2. Update the center.
- // center = 1/2 * lattice_param_vec + 1/2 * (Average(param_per_cluster) -
- // Average(dual))
- center.assign(param_size, 0);
- for (int ii = 0; ii < num_clusters; ++ii) {
- const std::vector& dual = duals[ii];
- const std::vector& param = param_per_cluster[ii];
- for (int jj = 0; jj < param_size; ++jj) {
- center[jj] += (param[jj] - dual[jj]);
- }
- }
- for (int ii = 0; ii < param_size; ++ii) {
- center[ii] *= average_scale;
- center[ii] += 0.5 * lattice_param_vec[ii];
- }
-
- // Step 3. Update the dual and residual
- residual = 0;
- for (int ii = 0; ii < num_clusters; ++ii) {
- std::vector& dual = duals[ii];
- const std::vector& param = param_per_cluster[ii];
- for (int jj = 0; jj < param_size; ++jj) {
- const Dtype diff = center[jj] - param[jj];
- dual[jj] += diff;
- residual += std::abs(diff);
- }
- }
-
- ++iter;
- if (iter > max_iter_) {
- break;
- }
- }
- return Status::OK();
-}
-
-} // namespace lattice
-} // namespace tensorflow
-
-#endif // TENSORFLOW_LATTICE_CC_KERNELS_MONOTONIC_LATTICE_PROJECTIONS_H_
diff --git a/tensorflow_lattice/cc/kernels/monotonic_lattice_projections_test.cc b/tensorflow_lattice/cc/kernels/monotonic_lattice_projections_test.cc
deleted file mode 100644
index 18e8525..0000000
--- a/tensorflow_lattice/cc/kernels/monotonic_lattice_projections_test.cc
+++ /dev/null
@@ -1,135 +0,0 @@
-/* Copyright 2017 The TensorFlow Lattice Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow_lattice/cc/kernels/monotonic_lattice_projections.h"
-
-#include
-
-#include "tensorflow_lattice/cc/lib/lattice_structure.h"
-#include "tensorflow/core/lib/core/status_test_util.h"
-#include "tensorflow/core/lib/strings/str_util.h"
-#include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/platform/test.h"
-
-namespace tensorflow {
-namespace lattice {
-namespace {
-TEST(MonotoneLatticeProjectorErrorTest, ProjectionWithNullptr) {
- LatticeStructure lattice_structure(/*lattice_sizes=*/{2, 2});
- MonotoneLatticeProjector projector(lattice_structure,
- /*monotone_dimensions=*/{});
- const Status s =
- projector.Project(/*lattice_param_vec=*/{0, 1, 2, 3}, nullptr);
- EXPECT_EQ(s.code(), error::INVALID_ARGUMENT);
-}
-
-TEST(MonotoneLatticeProjectorErrorTest, ProjectionWithWrongInputDimension) {
- LatticeStructure lattice_structure(/*lattice_sizes=*/{2, 2});
- MonotoneLatticeProjector projector(lattice_structure,
- /*monotone_dimensions=*/{});
- std::vector output(4, 0.0);
- const Status s = projector.Project(/*lattice_param_vec=*/{0, 1, 2}, &output);
- EXPECT_EQ(s.code(), error::INVALID_ARGUMENT);
-}
-
-TEST(MonotoneLatticeProjectorErrorTest, ProjectionWithWrongOutputDimension) {
- LatticeStructure lattice_structure(/*lattice_sizes=*/{2, 2});
- MonotoneLatticeProjector projector(lattice_structure,
- /*monotone_dimensions=*/{});
- std::vector output(3, 0.0);
- const Status s =
- projector.Project(/*lattice_param_vec=*/{0, 1, 2, 3}, &output);
- EXPECT_EQ(s.code(), error::INVALID_ARGUMENT);
-}
-
-// The fixture for testing MonotoneLatticeProjector.
-class MonotoneLatticeProjectorTest : public ::testing::Test {
- protected:
- void CheckProjection(
- const std::vector& lattice_sizes,
- const std::vector& monotone_dimensions,
- const std::vector& lattice_param_vec,
- const std::vector& expected_projected_lattice_param_vec) {
- LatticeStructure lattice_structure(lattice_sizes);
- MonotoneLatticeProjector projector(lattice_structure,
- monotone_dimensions, kEpsilon);
- std::vector projected_lattice_param_vec(lattice_param_vec.size());
- TF_ASSERT_OK(
- projector.Project(lattice_param_vec, &projected_lattice_param_vec));
- LOG(INFO) << "lattice param: " << str_util::Join(lattice_param_vec, ",");
- LOG(INFO) << "Expected projected lattice param: "
- << str_util::Join(expected_projected_lattice_param_vec, ",");
- LOG(INFO) << "Projected lattice param: "
- << str_util::Join(projected_lattice_param_vec, ",");
-
- ASSERT_EQ(projected_lattice_param_vec.size(),
- expected_projected_lattice_param_vec.size());
- for (int ii = 0; ii < expected_projected_lattice_param_vec.size(); ++ii) {
- EXPECT_NEAR(expected_projected_lattice_param_vec[ii],
- projected_lattice_param_vec[ii], kEpsilon);
- }
- }
-
- private:
- const float kEpsilon = 1e-5;
-};
-
-TEST_F(MonotoneLatticeProjectorTest, ProjectToNothing) {
- CheckProjection(
- /*lattice_sizes=*/{2, 2}, /*monotone_dimensions=*/{},
- /*lattice_param_vec=*/{3.0, 0.0, 2.0, 5.0},
- /*expected_projected_lattice_param_vec=*/{3.0, 0.0, 2.0, 5.0});
-}
-
-TEST_F(MonotoneLatticeProjectorTest, ProjectTo0thDimension) {
- CheckProjection(
- /*lattice_sizes=*/{2, 2}, /*monotone_dimensions=*/{0},
- /*lattice_param_vec=*/{3.0, 0.0, 2.0, 5.0},
- /*expected_projected_lattice_param_vec=*/{1.5, 1.5, 2.0, 5.0});
-}
-
-TEST_F(MonotoneLatticeProjectorTest, ProjectTo1stDimension) {
- CheckProjection(
- /*lattice_sizes=*/{2, 2}, /*monotone_dimensions=*/{1},
- /*lattice_param_vec=*/{3.0, 0.0, 2.0, 5.0},
- /*expected_projected_lattice_param_vec=*/{2.5, 0.0, 2.5, 5.0});
-}
-
-TEST_F(MonotoneLatticeProjectorTest, ProjectToAllDimensions) {
- CheckProjection(
- /*lattice_sizes=*/{2, 2}, /*monotone_dimensions=*/{0, 1},
- /*lattice_param_vec=*/{3.0, 0.0, 2.0, 5.0},
- /*expected_projected_lattice_param_vec=*/{1.5, 1.5, 2.0, 5.0});
-}
-
-TEST_F(MonotoneLatticeProjectorTest, ProjectThreeByTwoLatticeToAllDimensions) {
- CheckProjection(
- /*lattice_sizes=*/{3, 2}, /*monotone_dimensions=*/{0, 1},
- /*lattice_param_vec=*/{3.0, 1.0, 0.0, 0.0, 2.0, 5.0},
- /*expected_projected_lattice_param_vec=*/{1.0, 1.0, 1.0, 1.0, 2.0, 5.0});
-}
-
-TEST_F(MonotoneLatticeProjectorTest,
- ProjectTwoByTwoByTwoLatticeToAllDimensions) {
- CheckProjection(
- /*lattice_sizes=*/{2, 2, 2},
- /*monotone_dimensions=*/{0, 1, 2},
- /*lattice_param_vec=*/{0.44, 0.3, 0.12, 3.33, 3.0, 0.0, 2.0, 5.0},
- /*expected_projected_lattice_param_vec=*/{0.28, 0.3, 0.28, 3.33, 1.5, 1.5,
- 2.0, 5.0});
-}
-
-} // namespace
-} // namespace lattice
-} // namespace tensorflow
diff --git a/tensorflow_lattice/cc/kernels/monotonic_projection_kernel.cc b/tensorflow_lattice/cc/kernels/monotonic_projection_kernel.cc
deleted file mode 100644
index 018d9b9..0000000
--- a/tensorflow_lattice/cc/kernels/monotonic_projection_kernel.cc
+++ /dev/null
@@ -1,93 +0,0 @@
-/* Copyright 2017 The TensorFlow Lattice Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include
-#include
-#include
-
-#include "tensorflow_lattice/cc/kernels/monotonic_projections.h"
-#include "tensorflow/core/framework/op.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/framework/types.pb.h"
-
-namespace tensorflow {
-namespace lattice {
-
-namespace {
-
-template
-bool CmpLesserOrEqual(const Dtype a, const Dtype b) {
- return a <= b;
-}
-
-template
-bool CmpGreaterOrEqual(const Dtype a, const Dtype b) {
- return a >= b;
-}
-
-} // namespace
-
-template
-class MonotonicProjectionOpKernel : public OpKernel {
- public:
- explicit MonotonicProjectionOpKernel(OpKernelConstruction* context)
- : OpKernel(context) {}
-
- void Compute(OpKernelContext* context) override {
- const Tensor& values_tensor = context->input(0);
- const Tensor& increasing_tensor = context->input(1);
-
- OP_REQUIRES(
- context, values_tensor.dims() == 1,
- errors::InvalidArgument("values must have dims=1, got values.dims=",
- values_tensor.dims()));
- OP_REQUIRES(context, increasing_tensor.dims() == 0,
- errors::InvalidArgument(
- "increasing must be a boolean scalar, got increasing.dims=",
- increasing_tensor.dims()));
- OP_REQUIRES(
- context, increasing_tensor.dtype() == DT_BOOL,
- errors::InvalidArgument(
- "increasing must be a boolean scalar, got increasing.dtype=",
- DataType_Name(increasing_tensor.dtype())));
-
- Tensor* monotonic_tensor = nullptr;
- OP_REQUIRES_OK(
- context,
- context->allocate_output(0, values_tensor.shape(), &monotonic_tensor));
-
- // Copy the current non-monotonic values and project them to monotonicity.
- *monotonic_tensor = values_tensor;
- if (increasing_tensor.scalar()()) {
- TensorVectorMonotonicProjection(monotonic_tensor->vec(),
- CmpLesserOrEqual);
- } else {
- TensorVectorMonotonicProjection(monotonic_tensor->vec(),
- CmpGreaterOrEqual);
- }
- }
-};
-
-REGISTER_KERNEL_BUILDER(Name("MonotonicProjection")
- .Device(DEVICE_CPU)
- .TypeConstraint("Dtype"),
- MonotonicProjectionOpKernel);
-REGISTER_KERNEL_BUILDER(Name("MonotonicProjection")
- .Device(DEVICE_CPU)
- .TypeConstraint("Dtype"),
- MonotonicProjectionOpKernel);
-
-} // namespace lattice
-} // namespace tensorflow
diff --git a/tensorflow_lattice/cc/kernels/monotonic_projections.h b/tensorflow_lattice/cc/kernels/monotonic_projections.h
deleted file mode 100644
index 545ae6d..0000000
--- a/tensorflow_lattice/cc/kernels/monotonic_projections.h
+++ /dev/null
@@ -1,161 +0,0 @@
-/* Copyright 2017 The TensorFlow Lattice Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-// Functions that calculate monotonic projections.
-#ifndef TENSORFLOW_LATTICE_CC_KERNELS_MONOTONIC_PROJECTIONS_H_
-#define TENSORFLOW_LATTICE_CC_KERNELS_MONOTONIC_PROJECTIONS_H_
-
-#include
-
-#include "tensorflow/core/framework/tensor.h"
-
-namespace tensorflow {
-namespace lattice {
-
-// Converts a vector to a non-strictly monotonic vector that minimizes squared
-// distance to original vector values.
-//
-// monotonic_cmp is the comparison function that defines the direction of
-// the monotonicity. monotonic_cmp(a,b) should return true if a followed
-// by b is considered monotonic (equal values should always be considered
-// monotonic). monotonic_cmp should be transitive and
-// monotonic_cmp(a,b) && monotonic_cmp(b,a) iff a == b.
-template
-std::vector VectorMonotonicProjection(const std::vector& input,
- const CmpFn monotonic_cmp);
-
-// Converts a Tensor vector to a non-strictly monotonic vector that minimizes
-// squared distance to original vector values.
-//
-// monotonic_cmp is the comparison function that defines the direction of
-// the monotonicity. monotonic_cmp(a,b) should return true if a followed
-// by b is considered monotonic (equal values should always be considered
-// monotonic). monotonic_cmp should be transitive and
-// monotonic_cmp(a,b) && monotonic_cmp(b,a) iff a == b.
-template
-void TensorVectorMonotonicProjection(typename TTypes::Vec values,
- const CmpFn monotonic_cmp);
-
-// Converts a vector to a non-strictly monotonic vector that minimizes squared
-// distance to original vector values.
-//
-// Given a vector, input, it finds a non-strictly monotonic vector, output, such
-// that:
-//
-// 1. cmp_fn(output[i], output[i + 1]) == true for all 0 <= i < n -1
-// (e.g., output[0] <= output[1] <= ... <= output[n -1])
-// 2. minimizes || input - output ||_2
-//
-// This is a implementation special case of pool adjacent violators (PAV)
-// algorithm.
-//
-// To use it one provides a comparison function (that defines the desired
-// monotonicity direction) and Insert() one value at a time, in order.
-//
-// In the end one can project the monotonic vector into a std::vector or
-// directly into a Tensor vector.
-template
-class MonotonicProjector {
- public:
- // size is the size of the vector to be projected to monotonicity.
- // monotonic_cmp is the comparison function that defines the direction of
- // the monotonicity. monotonic_cmp(a,b) should return true if a followed
- // by b is considered monotonic (equal values should always be considered
- // monotonic). monotonic_cmp should be transitive and
- // monotonic_cmp(a,b) && monotonic_cmp(b,a) iff a == b.
- explicit MonotonicProjector(const int size, const CmpFn monotonic_cmp)
- : size_(size), monotonic_cmp_(monotonic_cmp) {
- pool_list_.reserve(size);
- }
-
- // Insert value to end of pool list keeping list monotonic according to
- // monotonic_cmp_.
- void Insert(Dtype value) {
- Pool new_pool{1, value, value};
- // While new_pool wouldn't be properly monotonic, merge the pool with the
- // previous one.
- while (!pool_list_.empty() &&
- !monotonic_cmp_(pool_list_.back().mean, new_pool.mean)) {
- // If last pool would break monotonicity,
- new_pool.size += pool_list_.back().size;
- new_pool.sum += pool_list_.back().sum;
- new_pool.mean = new_pool.sum / new_pool.size;
- pool_list_.pop_back();
- }
- pool_list_.push_back(new_pool);
- }
-
- // Copies monotonic projection to Tensor vector.
- void ProjectToTensorVector(typename TTypes::Vec output) {
- int output_index = 0;
- for (const auto& pool : pool_list_) {
- for (const int limit = output_index + pool.size; output_index < limit;
- ++output_index) {
- output(output_index) = pool.mean;
- }
- }
- }
-
- // Returns monotonic projection as vector.
- std::vector ProjectToVector() {
- std::vector output(size_);
- int output_index = 0;
- for (const auto& pool : pool_list_) {
- for (const int limit = output_index + pool.size; output_index < limit;
- ++output_index) {
- output[output_index] = pool.mean;
- }
- }
- return output;
- }
-
- private:
- struct Pool {
- int size; // Number of elements in pool.
- Dtype sum, mean; // Sum and mean of all values in pool.
- };
-
- const int size_;
- std::vector pool_list_;
- const CmpFn monotonic_cmp_;
-};
-
-// Implementation details
-
-// START_SKIP_DOXYGEN
-template
-std::vector VectorMonotonicProjection(const std::vector& input,
- const CmpFn monotonic_cmp) {
- MonotonicProjector projector(input.size(), monotonic_cmp);
- for (const Dtype value : input) {
- projector.Insert(value);
- }
- return projector.ProjectToVector();
-}
-
-template
-void TensorVectorMonotonicProjection(typename TTypes::Vec values,
- const CmpFn monotonic_cmp) {
- MonotonicProjector projector(values.size(), monotonic_cmp);
- for (int i = 0; i < values.size(); ++i) {
- projector.Insert(values(i));
- }
- projector.ProjectToTensorVector(values);
-}
-// END_SKIP_DOXYGEN
-
-} // namespace lattice
-} // namespace tensorflow
-
-#endif // TENSORFLOW_LATTICE_CC_KERNELS_MONOTONIC_PROJECTIONS_H_
diff --git a/tensorflow_lattice/cc/kernels/pwl_indexing_calibrator_kernels.cc b/tensorflow_lattice/cc/kernels/pwl_indexing_calibrator_kernels.cc
deleted file mode 100644
index aab1669..0000000
--- a/tensorflow_lattice/cc/kernels/pwl_indexing_calibrator_kernels.cc
+++ /dev/null
@@ -1,759 +0,0 @@
-/* Copyright 2017 The TensorFlow Lattice Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include
-#include
-#include
-#include
-
-#include "tensorflow/core/framework/common_shape_fns.h"
-#include "tensorflow/core/framework/op.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/framework/shape_inference.h"
-#include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/util/sparse/sparse_tensor.h"
-#include "tensorflow/core/util/work_sharder.h"
-
-namespace tensorflow {
-namespace lattice {
-
-namespace {
-
-// Maximum number of points used by interpolation. It may use up to 3 when it's
-// exactly on top of a keypoint input -- it returns also the left and right
-// keypoints inputs (indices). See explanation below on
-// FindExpandedInterpolation.
-constexpr int kMaxNumInterpolationPoints = 3;
-
-// Changed with PwlSetDebugMode function. This variable forces each row of a
-// batch to be processed by a separate worker, only used for testing.
-bool test_force_split = false;
-
-} // namespace
-
-extern void PwlSetTestMode(bool split_batches);
-void PwlSetTestMode(const bool split_batches) {
- test_force_split = split_batches;
-}
-
-// Helper struct that holds all information needed to resolve one interpolation:
-// the number of consecutive points used (num_points), the index of the first
-// one (lower_index) the associated weights -- not used in every case.
-template
-struct InterpolationPoints {
- int num_points;
- int64_t lower_index;
- Dtype weights[kMaxNumInterpolationPoints];
-};
-
-namespace {
-
-// Find the interpolation points, but _not the weights_, for the given
-// uncalibrated value and keypoints inputs (kp_inputs).
-// The interpolation will be between kp_inputs[lower_index] and
-// kp_inputs[lower_index + 1]. Except outside the edges or if x (uncalibrated)
-// is exactly on top of a keypoint, in which case the function returns 1 point.
-// It uses a simple binary-search, so it is O(log(|kp_inputs|)).
-template
-InterpolationPoints FindInterpolationPoints(
- const Dtype uncalibrated,
- const typename TTypes::Vec& kp_inputs) {
- if (uncalibrated <= kp_inputs(0)) {
- return InterpolationPoints{1, 0};
- }
- const int64_t kp_inputs_last_idx = static_cast(kp_inputs.size() - 1);
- if (uncalibrated >= kp_inputs(kp_inputs_last_idx)) {
- return InterpolationPoints{1, kp_inputs_last_idx};
- }
-
- // Binary search the keypoints inputs.
- int64_t min_idx = 0, max_idx = kp_inputs.size();
- while (max_idx > min_idx + 1) {
- const int64_t idx = (max_idx + min_idx) / 2;
- const Dtype value = kp_inputs(idx);
- if (uncalibrated == value) {
- return InterpolationPoints{1, idx};
- }
- if (uncalibrated > value) {
- min_idx = idx;
- } else {
- max_idx = idx;
- }
- }
-
- // Two points, where lower_index is min_idx.
- return InterpolationPoints{2, min_idx};
-}
-
-// Find interpolations points and associated weights for the given
-// uncalibrated value and keypoints inputs (kp_inputs).
-// Returns 1 interpolation point if uncalibrated is exactly on top of an
-// input keypoint (or if beyond the edges), or 2 if in between two
-// keypoints.
-// See FindInterpolationPoints.
-template
-InterpolationPoints FindInterpolationPointsWithWeights(
- const Dtype uncalibrated,
- const typename TTypes::Vec& kp_inputs) {
- // Get points an calculates weights.
- InterpolationPoints