[build] buildlet: support IAP tunnels for connecting to GCE VMs

72 views

Skip to first unread message

Heschi Kreinick (Gerrit)

unread,

Nov 19, 2021, 1:35:56 PM11/19/21

to goph...@pubsubhelper.golang.org, golang-...@googlegroups.com, Go Bot, Carlos Amedee, Brad Fitzpatrick, Dmitri Shuralyov, Alexander Rakoczy, golang-co...@googlegroups.com

Heschi Kreinick submitted this change.

View Change



1 is the latest approved patch-set.
The change was submitted with unreviewed changes in the following files:

```
The name of the file: buildlet/buildlet.go
Insertions: 1, Deletions: 1.

@@ -76,7 +76,7 @@
 	// on port 80 or 443 before creating a buildlet client.
 	SkipEndpointVerification bool
 
-	// Use an IAP tunnel to connect to buildlets on GCP.
+	// UseIAPTunnel uses an IAP tunnel to connect to buildlets on GCP.
 	UseIAPTunnel bool
 }
 
```

Approvals:
  Carlos Amedee: Looks good to me, approved
  Heschi Kreinick: Trusted; Run TryBots
  Go Bot: TryBots succeeded

buildlet: support IAP tunnels for connecting to GCE VMs

Now that we're not allocating external IP addresses to our VMs, you
can't get to them directly from outside our GCP project. The way to fix
that is to use an IAP tunnel. Sadly, there is no API for creating an IAP
tunnel other than gcloud; we have to shell out to it.

Change-Id: I577734aae669a952705276f88f878f040a4b3bf0
Reviewed-on: https://go-review.googlesource.com/c/build/+/364414
Trust: Heschi Kreinick <hes...@google.com>
Run-TryBot: Heschi Kreinick <hes...@google.com>
TryBot-Result: Go Bot <go...@golang.org>
Reviewed-by: Carlos Amedee <car...@golang.org>
---
M buildlet/buildlet.go
M buildlet/gce.go
M cmd/debugnewvm/debugnewvm.go
3 files changed, 86 insertions(+), 1 deletion(-)

diff --git a/buildlet/buildlet.go b/buildlet/buildlet.go
index 1696303..d29b76f 100644
--- a/buildlet/buildlet.go
+++ b/buildlet/buildlet.go
@@ -75,6 +75,9 @@
 	// SkipEndpointVerification does not verify that the builder is listening
 	// on port 80 or 443 before creating a buildlet client.
 	SkipEndpointVerification bool
+
+	// UseIAPTunnel uses an IAP tunnel to connect to buildlets on GCP.
+	UseIAPTunnel bool
 }
 
 // buildletClient returns a buildlet client configured to speak to a VM via the buildlet
diff --git a/buildlet/gce.go b/buildlet/gce.go
index 952d11f..3c12818 100644
--- a/buildlet/gce.go
+++ b/buildlet/gce.go
@@ -9,6 +9,9 @@
 	"errors"
 	"fmt"
 	"log"
+	"net"
+	"os"
+	"os/exec"
 	"sort"
 	"strings"
 	"sync"
@@ -254,7 +257,64 @@
 	if opts.OnGotInstanceInfo != nil {
 		opts.OnGotInstanceInfo(inst)
 	}
-	return buildletClient(ctx, buildletURL, ipPort, &opts)
+	var closeFuncs []func()
+	if opts.UseIAPTunnel {
+		localPort, closeFunc, err := createIAPTunnel(ctx, inst)
+		if err != nil {
+			return nil, fmt.Errorf("creating IAP tunnel: %v", err)
+		}
+		buildletURL = "http://localhost:" + localPort
+		ipPort = "127.0.0.1:" + localPort
+		closeFuncs = append(closeFuncs, closeFunc)
+	}
+	client, err := buildletClient(ctx, buildletURL, ipPort, &opts)
+	if err != nil {
+		return nil, err
+	}
+	client.closeFuncs = append(client.closeFuncs, closeFuncs...)
+	return client, nil
+}
+
+func createIAPTunnel(ctx context.Context, inst *compute.Instance) (string, func(), error) {
+	// Allocate a local listening port.
+	ln, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		return "", nil, err
+	}
+	localAddr := ln.Addr().(*net.TCPAddr)
+	ln.Close()
+	// Start the gcloud command. For some reason, when gcloud is run with a
+	// pipe for stdout, it doesn't log the success message, so we can only
+	// check for success empirically.
+	tunnelCmd := exec.CommandContext(ctx,
+		"gcloud", "compute", "start-iap-tunnel", "--iap-tunnel-disable-connection-check",
+		"--zone", inst.Zone, inst.Name, "80", "--local-host-port", localAddr.String())
+	tunnelCmd.Stderr = os.Stderr
+	tunnelCmd.Stdout = os.Stdout
+	if err := tunnelCmd.Start(); err != nil {
+		return "", nil, err
+	}
+	// Start the process. Either it's going to fail to start after a bit, or
+	// it'll start listening on its port. Because we told it not to check the
+	// connection above, the connections won't be functional, but we can dial.
+	errc := make(chan error, 1)
+	go func() { errc <- tunnelCmd.Wait() }()
+	for start := time.Now(); time.Since(start) < 60*time.Second; time.Sleep(5 * time.Second) {
+		// Check if the server crashed.
+		select {
+		case err := <-errc:
+			return "", nil, err
+		default:
+		}
+		// Check if it's healthy.
+		conn, err := net.DialTCP("tcp", nil, localAddr)
+		if err == nil {
+			conn.Close()
+			kill := func() { tunnelCmd.Process.Kill() }
+			return fmt.Sprint(localAddr.Port), kill, nil
+		}
+	}
+	return "", nil, fmt.Errorf("iap tunnel startup timed out")
 }
 
 // DestroyVM sends a request to delete a VM. Actual VM description is
diff --git a/cmd/debugnewvm/debugnewvm.go b/cmd/debugnewvm/debugnewvm.go
index 1c26ffa..b04d7e7 100644
--- a/cmd/debugnewvm/debugnewvm.go
+++ b/cmd/debugnewvm/debugnewvm.go
@@ -41,6 +41,8 @@
 	makeOnly = flag.Bool("make-only", false, "if a --run-build builder name is given, this controls whether make.bash or all.bash is run")
 	buildRev = flag.String("rev", "master", "if --run-build is specified, the git hash or branch name to build")
 
+	useIAPTunnel = flag.Bool("use-iap-tunnel", true, "use an IAP tunnel to connect to GCE builders")
+
 	awsKeyID     = flag.String("aws-key-id", "", "if the builder runs on aws then key id is required. If executed on GCE, it will be retrieved from secrets.")
 	awsAccessKey = flag.String("aws-access-key", "", "if the builder runs on aws then the access key is required. If executed on GCE, it will be retrieved from secrets.")
 	awsRegion    = flag.String("aws-region", "", "if non-empty and the requested builder is an EC2 instance, force an EC2 region.")
@@ -293,6 +295,7 @@
 			}
 			log.Printf("buildlet probe: %s", res.Status)
 		},
+		UseIAPTunnel: *useIAPTunnel,
 	})
 }

To view, visit change 364414. To unsubscribe, or for help writing mail filters, visit settings.

Reply all

Reply to author

Forward

0 new messages