Skip to content

Commit

Permalink
Simplify and improve fastprep.mk
Browse files Browse the repository at this point in the history
There is no need in building Tensorflow from scratch, fastprep successfully links with the Python wrapper from pip and system protobuf v3. Tested this on Ubuntu.
  • Loading branch information
vmarkovtsev committed Mar 2, 2017
1 parent bc70271 commit 4cdfc96
Showing 1 changed file with 26 additions and 53 deletions.
79 changes: 26 additions & 53 deletions swivel/fastprep.mk
Original file line number Diff line number Diff line change
Expand Up @@ -16,50 +16,16 @@
# limitations under the License.



# This makefile builds "fastprep", a faster version of prep.py that can be used
# to build training data for Swivel. Building "fastprep" is a bit more
# involved: you'll need to pull and build the Tensorflow source, and then build
# and install compatible protobuf software. We've tested this with Tensorflow
# version 0.7.
#
# = Step 1. Pull and Build Tensorflow. =
#
# These instructions are somewhat abridged; for pre-requisites and the most
# up-to-date instructions, refer to:
#
# <https://www.tensorflow.org/versions/r0.7/get_started/os_setup.html#installing-from-sources>
#
# To build the Tensorflow components required for "fastpret", you'll need to
# install Bazel, Numpy, Swig, and Python development headers as described in at
# the above URL. Run the "configure" script as appropriate for your
# environment and then build the "build_pip_package" target:
#
# bazel build -c opt [--config=cuda] //tensorflow/tools/pip_package:build_pip_package
#
# This will generate the Tensorflow headers and libraries necessary for
# "fastprep".
#
# to build training data for Swivel.
#
# = Step 2. Build and Install Compatible Protobuf Libraries =
# = Step 1. Install protobuf v3 =
#
# "fastprep" also needs compatible protocol buffer libraries, which you can
# build from the protobuf implementation included with the Tensorflow
# distribution:
# Ubuntu 16.10+: sudo apt install libprotobuf-dev
# Ubuntu 16.04: https://launchpad.net/~maarten-fonville/+archive/ubuntu/ppa + replace xenial with yakkety in /etc/apt/sources.list.d/maarten-fonville-ubuntu-ppa-xenial.list
# macOS: brew install protobuf
#
# cd ${TENSORFLOW_SRCDIR}/google/protobuf
# ./autogen.sh
# ./configure --prefix=${HOME} # ...or whatever
# make
# make install # ...or maybe "sudo make install"
#
# This will install the headers and libraries appropriately.
#
#
# = Step 3. Build "fastprep". =
#
# Finally modify this file (if necessary) to update PB_DIR and TF_DIR to refer
# to appropriate locations, and:
# = Step 2. Build "fastprep". =
#
# make -f fastprep.mk
#
Expand All @@ -68,20 +34,27 @@
# matrices and other files necessary to train a Swivel matrix.


# The root directory where the Google Protobuf software is installed.
# Alternative locations might be "/usr" or "/usr/local".
PB_DIR=$(HOME)
CXXFLAGS=-std=c++11 -march=native -g -O2 -flto -Wall -I.
LDLIBS=-lprotobuf -pthread -lm

FETCHER=curl -L -o
TF_URL=https://github.com/tensorflow/tensorflow/raw/master
PROTOC=protoc


%.proto: tensorflow/core/example
$(FETCHER) $@ $(TF_URL)/$@

# Assuming you've got the Tensorflow source unpacked and built in ${HOME}/src:
TF_DIR=$(HOME)/src/tensorflow
%.pb.cc: %.proto
$(PROTOC) --cpp_out=. $<

PB_INCLUDE=$(PB_DIR)/include
TF_INCLUDE=$(TF_DIR)/bazel-genfiles
CXXFLAGS=-std=c++11 -m64 -mavx -g -Ofast -Wall -I$(TF_INCLUDE) -I$(PB_INCLUDE)
fastprep: fastprep.cc tensorflow/core/example/feature.pb.cc tensorflow/core/example/example.pb.cc

PB_LIBDIR=$(PB_DIR)/lib
TF_LIBDIR=$(TF_DIR)/bazel-bin/tensorflow/core
LDFLAGS=-L$(TF_LIBDIR) -L$(PB_LIBDIR)
LDLIBS=-lprotos_all_cc -lprotobuf -lpthread -lm
tensorflow/core/example:
@mkdir -p tensorflow/core/example

fastprep: fastprep.cc
clean:
@rm -f fastprep

mrproper: clean
@rm -rf tensorflow

0 comments on commit 4cdfc96

Please sign in to comment.