Change in bazel[master]: Make compile.sh use pregenerated protoc output, if present

222 views
Skip to first unread message

Klaus Aehlig (Gerrit)

unread,
Oct 17, 2016, 11:31:41 AM10/17/16
to bazel-de...@googlegroups.com
Klaus Aehlig has uploaded a new change for review. (
https://bazel-review.googlesource.com/6731


Change subject: Make compile.sh use pregenerated protoc output, if present
......................................................................

Make compile.sh use pregenerated protoc output, if present

Change ./compile.sh to look for pregenerated protoc output and use it,
if present. If not, insist on PROTOC and GRPC_JAVA_PLUGIN to be specified
via the environment. In this way compile.sh does not depend on committed
binaries any more.

Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
---
M scripts/bootstrap/compile.sh
1 file changed, 23 insertions(+), 48 deletions(-)



diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh
index d897a4b..83e25a8 100755
--- a/scripts/bootstrap/compile.sh
+++ b/scripts/bootstrap/compile.sh
@@ -42,44 +42,17 @@
linux)
# JAVA_HOME must point to a Java installation.
JAVA_HOME="${JAVA_HOME:-$(readlink -f $(which javac) |
sed 's_/bin/javac__')}"
- if [ "${MACHINE_IS_64BIT}" = 'yes' ]; then
- if [ "${MACHINE_IS_Z}" = 'yes' ]; then
- PROTOC=${PROTOC:-third_party/protobuf/protoc-linux-s390x_64.exe}
-
GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-linux-s390x_64.exe}
- else
- PROTOC=${PROTOC:-third_party/protobuf/protoc-linux-x86_64.exe}
-
GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-linux-x86_64.exe}
- fi
- else
- if [ "${MACHINE_IS_ARM}" = 'yes' ]; then
- PROTOC=${PROTOC:-third_party/protobuf/protoc-linux-arm32.exe}
- else
- PROTOC=${PROTOC:-third_party/protobuf/protoc-linux-x86_32.exe}
-
GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-linux-x86_32.exe}
- fi
- fi
;;

freebsd)
# JAVA_HOME must point to a Java installation.
JAVA_HOME="${JAVA_HOME:-/usr/local/openjdk8}"
- # Note: the linux protoc binary works on freebsd using linux emulation.
- # We choose the 32-bit version for maximum compatiblity since 64-bit
- # linux binaries are only supported in FreeBSD-11.
- PROTOC=${PROTOC:-third_party/protobuf/protoc-linux-x86_32.exe}
-
GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-linux-x86_32.exe}
;;

darwin)
if [[ -z "$JAVA_HOME" ]]; then
JAVA_HOME="$(/usr/libexec/java_home -v ${JAVA_VERSION}+ 2> /dev/null)"
\
|| fail "Could not find JAVA_HOME, please ensure a JDK (version
${JAVA_VERSION}+) is installed."
- fi
- if [ "${MACHINE_IS_64BIT}" = 'yes' ]; then
- PROTOC=${PROTOC:-third_party/protobuf/protoc-osx-x86_64.exe}
-
GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-osx-x86_64.exe}
- else
- PROTOC=${PROTOC:-third_party/protobuf/protoc-osx-x86_32.exe}
fi
;;

@@ -89,21 +62,8 @@
PATHSEP=";"
# Find the latest available version of the SDK.
JAVA_HOME="${JAVA_HOME:-$(ls -d /c/Program\ Files/Java/jdk* | sort |
tail -n 1)}"
- # We do not use the JNI library on Windows.
- if [ "${MACHINE_IS_64BIT}" = 'yes' ]; then
- PROTOC=${PROTOC:-third_party/protobuf/protoc-windows-x86_64.exe}
-
GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-windows-x86_64.exe}
- else
- PROTOC=${PROTOC:-third_party/protobuf/protoc-windows-x86_32.exe}
-
GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-windows-x86_32.exe}
- fi
esac

-[[ -x "${PROTOC-}" ]] \
- || fail "Protobuf compiler not found in ${PROTOC-}"
-
-[[ -x "${GRPC_JAVA_PLUGIN-}" ]] \
- || fail "gRPC Java plugin not found in ${GRPC_JAVA_PLUGIN-}"

# Check that javac -version returns a upper version than $JAVA_VERSION.
get_java_version
@@ -183,14 +143,29 @@
}

if [ -z "${BAZEL_SKIP_JAVA_COMPILATION}" ]; then
- log "Compiling Java stubs for protocol buffers..."
- for f in $PROTO_FILES ; do
- run "${PROTOC}" -Isrc/main/protobuf/ \
-
-Isrc/main/java/com/google/devtools/build/lib/buildeventstream/proto/ \
- --java_out=${OUTPUT_DIR}/src \
- --plugin=protoc-gen-grpc="${GRPC_JAVA_PLUGIN-}" \
- --grpc_out=${OUTPUT_DIR}/src "$f"
- done
+
+ if [ -d derived/src/java ]
+ then
+ log "Using pre-generated java proto files"
+ mkdir -p "${OUTPUT_DIR}/src"
+ cp -r derived/src/java/* "${OUTPUT_DIR}/src"
+ else
+
+ [[ -x "${PROTOC-}" ]] \
+ || fail "Protobuf compiler not found in ${PROTOC-}"
+
+ [[ -x "${GRPC_JAVA_PLUGIN-}" ]] \
+ || fail "gRPC Java plugin not found in ${GRPC_JAVA_PLUGIN-}"
+
+ log "Compiling Java stubs for protocol buffers..."
+ for f in $PROTO_FILES ; do
+ run "${PROTOC}" -Isrc/main/protobuf/ \
+
-Isrc/main/java/com/google/devtools/build/lib/buildeventstream/proto/ \
+ --java_out=${OUTPUT_DIR}/src \
+ --plugin=protoc-gen-grpc="${GRPC_JAVA_PLUGIN-}" \
+ --grpc_out=${OUTPUT_DIR}/src "$f"
+ done
+ fi

java_compilation "Bazel
Java" "$DIRS" "$EXCLUDE_FILES" "$LIBRARY_JARS" "${OUTPUT_DIR}"


--
To view, visit https://bazel-review.googlesource.com/6731
To unsubscribe, visit https://bazel-review.googlesource.com/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
Gerrit-PatchSet: 1
Gerrit-Project: bazel
Gerrit-Branch: master
Gerrit-Owner: Klaus Aehlig <aeh...@google.com>

Damien Martin-guillerez (Gerrit)

unread,
Oct 17, 2016, 11:42:47 AM10/17/16
to Klaus Aehlig
Damien Martin-guillerez has posted comments on this change. (
https://bazel-review.googlesource.com/6731 )

Change subject: Make compile.sh use pregenerated protoc output, if present
......................................................................


Patch Set 2:

Before making this one, we need to first make sure we ship the distribution
artifact on ci (change to scripts/ci/build.sh).

Also we need to ensure PROTOC and GRPC_PLUGIN is set on CI + have
documentation. So this change is ok but we need more change before :)
--
To view, visit https://bazel-review.googlesource.com/6731
To unsubscribe, visit https://bazel-review.googlesource.com/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
Gerrit-PatchSet: 2
Gerrit-Project: bazel
Gerrit-Branch: master
Gerrit-Owner: Klaus Aehlig <aeh...@google.com>
Gerrit-Reviewer: Damien Martin-guillerez <dmar...@google.com>
Gerrit-HasComments: No

Klaus Aehlig (Gerrit)

unread,
Oct 17, 2016, 11:54:49 AM10/17/16
to Damien Martin-guillerez
Klaus Aehlig has posted comments on this change. (
https://bazel-review.googlesource.com/6731 )

Change subject: Make compile.sh use pregenerated protoc output, if present
......................................................................


Patch Set 1:

> Before making this one, we need to first make sure we ship the
> distribution artifact on ci (change to scripts/ci/build.sh).

That's precisely the comment I made on the previous patch.

> Also we need to ensure PROTOC and GRPC_PLUGIN is set on CI + have
> documentation.

Well, if we build from the distribution artifact, the whole point
is that we do *not* need PROTOC and GRPC_PLUGIN be set, as the
distribution artifact already contains all the generated sources.
compile.sh will only look for those variables, if the generated
files are not shipped.
--
To view, visit https://bazel-review.googlesource.com/6731
To unsubscribe, visit https://bazel-review.googlesource.com/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
Gerrit-PatchSet: 1
Gerrit-Project: bazel
Gerrit-Branch: master
Gerrit-Owner: Klaus Aehlig <aeh...@google.com>
Gerrit-Reviewer: Damien Martin-guillerez <dmar...@google.com>
Gerrit-Reviewer: Klaus Aehlig <aeh...@google.com>
Gerrit-HasComments: No

Damien Martin-guillerez (Gerrit)

unread,
Oct 17, 2016, 12:22:19 PM10/17/16
to Klaus Aehlig
Damien Martin-guillerez has posted comments on this change. (
https://bazel-review.googlesource.com/6731 )

Change subject: Make compile.sh use pregenerated protoc output, if present
......................................................................


Patch Set 2:

> Patch Set 1:

> > Before making this one, we need to first make sure we ship the
> > distribution artifact on ci (change to scripts/ci/build.sh).

> That's precisely the comment I made on the previous patch.

Why not have the scripts/ci/build.sh modification before this one?


> > Also we need to ensure PROTOC and GRPC_PLUGIN is set on CI + have
> > documentation.

> Well, if we build from the distribution artifact, the whole point
> is that we do *not* need PROTOC and GRPC_PLUGIN be set, as the
> distribution artifact already contains all the generated sources.
> compile.sh will only look for those variables, if the generated
> files are not shipped.

I mean developer documentation for people not using the distribution
artifact.
--
To view, visit https://bazel-review.googlesource.com/6731
To unsubscribe, visit https://bazel-review.googlesource.com/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
Gerrit-PatchSet: 2
Gerrit-Project: bazel
Gerrit-Branch: master
Gerrit-Owner: Klaus Aehlig <aeh...@google.com>

Klaus Aehlig (Gerrit)

unread,
Oct 19, 2016, 12:20:41 PM10/19/16
to Damien Martin-guillerez
Klaus Aehlig has posted comments on this change. (
https://bazel-review.googlesource.com/6731 )

Change subject: Make compile.sh use pregenerated protoc output, if present
......................................................................


Patch Set 2:

> Why not have the scripts/ci/build.sh modification before this one?

I'm not sure which modifications you're talking about, as all
building done in scripts/ci/build.sh is done from the git-repo,
not the source tar ball.
--
To view, visit https://bazel-review.googlesource.com/6731
To unsubscribe, visit https://bazel-review.googlesource.com/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
Gerrit-PatchSet: 2
Gerrit-Project: bazel
Gerrit-Branch: master
Gerrit-Owner: Klaus Aehlig <aeh...@google.com>

Klaus Aehlig (Gerrit)

unread,
Oct 21, 2016, 11:23:35 AM10/21/16
to Damien Martin-guillerez
Hello Damien Martin-guillerez,

I'd like you to reexamine a change. Please visit

https://bazel-review.googlesource.com/6731

to look at the new patch set (#4).

Change subject: Make compile.sh use pregenerated protoc output, if present
......................................................................

Make compile.sh use pregenerated protoc output, if present

Change ./compile.sh to look for pregenerated protoc output and use it,
if present. If not, insist on PROTOC and GRPC_JAVA_PLUGIN to be specified
via the environment. In this way compile.sh does not depend on committed
binaries any more.

Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
---
M scripts/bootstrap/compile.sh
1 file changed, 29 insertions(+), 48 deletions(-)


--
To view, visit https://bazel-review.googlesource.com/6731
To unsubscribe, visit https://bazel-review.googlesource.com/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
Gerrit-PatchSet: 4
Gerrit-Project: bazel
Gerrit-Branch: master
Gerrit-Owner: Klaus Aehlig <aeh...@google.com>

Klaus Aehlig (Gerrit)

unread,
Nov 11, 2016, 8:12:50 AM11/11/16
to Damien Martin-guillerez

Klaus Aehlig uploaded patch set #14 to Make compile.sh use pregenerated protoc output, if present.

View Change

Make compile.sh use pregenerated protoc output, if present

Change ./compile.sh to look for pregenerated protoc output and use it,
if present. If not, insist on PROTOC and GRPC_JAVA_PLUGIN to be specified
via the environment. In this way compile.sh does not depend on committed
binaries any more.

Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
---
M scripts/bootstrap/compile.sh
1 file changed, 29 insertions(+), 48 deletions(-)

To view, visit this change. To unsubscribe, visit settings.

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
Gerrit-PatchSet: 14
Gerrit-Project: bazel
Gerrit-Branch: master

Dmitry Lomov (Gerrit)

unread,
Nov 23, 2016, 12:03:01 PM11/23/16
to Klaus Aehlig, Damien Martin-guillerez

Dmitry Lomov uploaded patch set #19 to this change.

View Change

Make compile.sh use pregenerated protoc output, if present

Change ./compile.sh to look for pregenerated protoc output and use it,
if present. If not, insist on PROTOC and GRPC_JAVA_PLUGIN to be specified
via the environment. In this way compile.sh does not depend on committed
binaries any more.

--
Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
Reviewed-on: https://bazel-review.googlesource.com/#/c/6731
MOS_MIGRATED_REVID=140037537
---
M scripts/bootstrap/compile.sh
1 file changed, 29 insertions(+), 48 deletions(-)

To view, visit this change. To unsubscribe, visit settings.

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
Gerrit-Change-Number: 6731
Gerrit-PatchSet: 19
Gerrit-Project: bazel
Gerrit-Branch: master
Gerrit-Owner: Klaus Aehlig <aeh...@google.com>Gerrit-Reviewer: Damien Martin-guillerez <dmar...@google.com>Gerrit-Reviewer: Dmitry Lomov <dsl...@google.com>Gerrit-Reviewer: Klaus Aehlig <aeh...@google.com>

Dmitry Lomov (Gerrit)

unread,
Nov 23, 2016, 12:03:02 PM11/23/16
to Klaus Aehlig, Damien Martin-guillerez

Dmitry Lomov merged this change.

View Change

Make compile.sh use pregenerated protoc output, if present

Change ./compile.sh to look for pregenerated protoc output and use it,
if present. If not, insist on PROTOC and GRPC_JAVA_PLUGIN to be specified
via the environment. In this way compile.sh does not depend on committed
binaries any more.

--
Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
Reviewed-on: https://bazel-review.googlesource.com/#/c/6731
MOS_MIGRATED_REVID=140037537
---
M scripts/bootstrap/compile.sh
1 file changed, 29 insertions(+), 48 deletions(-)


diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh
index 0491d1c..1bf1e0c 100755
--- a/scripts/bootstrap/compile.sh
+++ b/scripts/bootstrap/compile.sh
@@ -42,44 +42,17 @@
 linux)
   # JAVA_HOME must point to a Java installation.
   JAVA_HOME="${JAVA_HOME:-$(readlink -f $(which javac) | sed 's_/bin/javac__')}"
-  if [ "${MACHINE_IS_64BIT}" = 'yes' ]; then
-    if [ "${MACHINE_IS_Z}" = 'yes' ]; then
-       PROTOC=${PROTOC:-third_party/protobuf/3.0.0/protoc-3.0.0-linux-s390x_64.exe}
-       GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-linux-s390x_64.exe}
-    else
-       PROTOC=${PROTOC:-third_party/protobuf/3.0.0/protoc-3.0.0-linux-x86_64.exe}
-       GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-linux-x86_64.exe}
-    fi
-  else
-    if [ "${MACHINE_IS_ARM}" = 'yes' ]; then
-      PROTOC=${PROTOC:-third_party/protobuf/3.0.0/protoc-3.0.0-linux-arm32.exe}
-    else
-      PROTOC=${PROTOC:-third_party/protobuf/3.0.0/protoc-3.0.0-linux-x86_32.exe}
-      GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-linux-x86_32.exe}
-    fi
-  fi
   ;;
 
 freebsd)
   # JAVA_HOME must point to a Java installation.
   JAVA_HOME="${JAVA_HOME:-/usr/local/openjdk8}"
-  # Note: the linux protoc binary works on freebsd using linux emulation.
-  # We choose the 32-bit version for maximum compatiblity since 64-bit
-  # linux binaries are only supported in FreeBSD-11.
-  PROTOC=${PROTOC:-third_party/protobuf/3.0.0/protoc-3.0.0-linux-x86_32.exe}
-  GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-linux-x86_32.exe}
   ;;
 
 darwin)
   if [[ -z "$JAVA_HOME" ]]; then
     JAVA_HOME="$(/usr/libexec/java_home -v ${JAVA_VERSION}+ 2> /dev/null)" \
       || fail "Could not find JAVA_HOME, please ensure a JDK (version ${JAVA_VERSION}+) is installed."
-  fi
-  if [ "${MACHINE_IS_64BIT}" = 'yes' ]; then
-    PROTOC=${PROTOC:-third_party/protobuf/3.0.0/protoc-3.0.0-osx-x86_64.exe}
-    GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-osx-x86_64.exe}
-  else
-    PROTOC=${PROTOC:-third_party/protobuf/3.0.0/protoc-3.0.0-osx-x86_32.exe}
   fi
   ;;
 
@@ -89,21 +62,8 @@
   PATHSEP=";"
   # Find the latest available version of the SDK.
   JAVA_HOME="${JAVA_HOME:-$(ls -d /c/Program\ Files/Java/jdk* | sort | tail -n 1)}"
-  # We do not use the JNI library on Windows.
-  if [ "${MACHINE_IS_64BIT}" = 'yes' ]; then
-    PROTOC=${PROTOC:-third_party/protobuf/3.0.0/protoc-3.0.0-windows-x86_64.exe}
-    GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-windows-x86_64.exe}
-  else
-    PROTOC=${PROTOC:-third_party/protobuf/3.0.0/protoc-3.0.0-windows-x86_32.exe}
-    GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.15.0-windows-x86_32.exe}
-  fi
 esac
 
-[[ -x "${PROTOC-}" ]] \
-    || fail "Protobuf compiler not found in ${PROTOC-}"
-
-[[ -x "${GRPC_JAVA_PLUGIN-}" ]] \
-    || fail "gRPC Java plugin not found in ${GRPC_JAVA_PLUGIN-}"
 
 # Check that javac -version returns a upper version than $JAVA_VERSION.
 get_java_version
@@ -183,14 +143,35 @@
 }
 
 if [ -z "${BAZEL_SKIP_JAVA_COMPILATION}" ]; then
-  log "Compiling Java stubs for protocol buffers..."
-  for f in $PROTO_FILES ; do
-    run "${PROTOC}" -Isrc/main/protobuf/ \
-        -Isrc/main/java/com/google/devtools/build/lib/buildeventstream/proto/ \
-        --java_out=${OUTPUT_DIR}/src \
-        --plugin=protoc-gen-grpc="${GRPC_JAVA_PLUGIN-}" \
-        --grpc_out=${OUTPUT_DIR}/src "$f"
-  done
+
+    if [ -d derived/src/java ]
+    then
+        log "Using pre-generated java proto files"
+        mkdir -p "${OUTPUT_DIR}/src"
+        cp -r derived/src/java/* "${OUTPUT_DIR}/src"
+    else
+
+        [ -n "${PROTOC}" ] \
+            || fail "Must specify PROTOC if not bootstrapping from the distribution artifact"
+
+        [ -n "${GRPC_JAVA_PLUGIN}" ] \
+            || fail "Must specify GRPC_JAVA_PLUGIN if not bootstrapping from the distribution artifact"
+
+        [[ -x "${PROTOC-}" ]] \
+            || fail "Protobuf compiler not found in ${PROTOC-}"
+
+        [[ -x "${GRPC_JAVA_PLUGIN-}" ]] \
+            || fail "gRPC Java plugin not found in ${GRPC_JAVA_PLUGIN-}"
+
+        log "Compiling Java stubs for protocol buffers..."
+        for f in $PROTO_FILES ; do
+            run "${PROTOC}" -Isrc/main/protobuf/ \
+                -Isrc/main/java/com/google/devtools/build/lib/buildeventstream/proto/ \
+                --java_out=${OUTPUT_DIR}/src \
+                --plugin=protoc-gen-grpc="${GRPC_JAVA_PLUGIN-}" \
+                --grpc_out=${OUTPUT_DIR}/src "$f"
+        done
+    fi
 
   java_compilation "Bazel Java" "$DIRS" "$EXCLUDE_FILES" "$LIBRARY_JARS" "${OUTPUT_DIR}"

To view, visit this change. To unsubscribe, visit settings.

Gerrit-MessageType: merged


Gerrit-Change-Id: I0d19a16535b606d2a6ab26ca349a7f9db8e224b7
Gerrit-Change-Number: 6731
Gerrit-PatchSet: 19

Gerrit-Project: bazel
Gerrit-Branch: master

Reply all
Reply to author
Forward
0 new messages