Capturing header information in har file without the use of net.lightbody.bmp.proxy package

1,166 views
Skip to first unread message

Brittany Jones

unread,
Jun 4, 2015, 4:53:24 PM6/4/15
to browserm...@googlegroups.com
I currently have an API that I have built in .Net C# to run browsermob-proxy which is similar to this one: https://github.com/AutomatedTester/AutomatedTester.BrowserMob . I'm trying to capture header information in my har files.
I'm aware that you have to set the captureHeaders boolean in order to get this to work. 
I see a lot of people doing something like this:

ProxyServer server = new ProxyServer(proxy_port);  // package net.lightbody.bmp.proxy

    server.start(); 
    server.setCaptureHeaders(true);

using the ProxyServer class within the net.lightbody.bmp.proxy package to initialize and start a new proxy server as they are writing their applications in Java and then calling the setCaptureHeaders function. However, I have created my own class in .Net called Proxy with all the functionality to start and stop the server that I need in it. Therefore I don't have the capability to use the setCaptureHeaders() function and set the boolean there.

I have been going through the browsermob-proxy java code trying to find out exactly how this functionality is being handled so that I can write a C# equivalent to capture the header information but it's not too clear for me. Does anyone know how I would incorporate this functionality into my .Net application? It would be easier if I was using Java then I could just do : package net.lightbody.bmp.proxy; and import what I need. 

 

je...@outlook.com

unread,
Jun 4, 2015, 10:13:44 PM6/4/15
to browserm...@googlegroups.com
How are you starting BrowserMob Proxy from your C# code? Are you running the executable and using the REST API to control it?

Brittany Jones

unread,
Jun 5, 2015, 11:19:59 AM6/5/15
to browserm...@googlegroups.com
The API was built to use BrowserMob proxy with Selenium.

This is how I'm starting the BrowserMob Proxy from my code:

// Start the proxy. (This happens when I start executing a test in selenium but before the web browser actually fires up)
            if (!string.IsNullOrWhiteSpace(DefaultValues.GetValue("ProxyPath")))
            {
                Proxy server = new Proxy(DefaultValues.GetValue("ProxyPath")); - Here I pass in the path to the browsermob-proxy batch file
                server.Start();

                WebTest.ServerProxy = server;
                Client client = WebTest.ServerProxy.CreateProxy();
                client.NewHar("performance"); - Here I'm creating the request for a new har file
                WebTest.Client = client;
            }

My proxy class that I'm creating a server instance of looks like this:

public class Proxy
    {
        private const string Host = "localhost";
        private readonly int port;
        private readonly string path = string.Empty;
        private Process serverProcess;

        public Proxy(string path) : this(path, 81)
        { 
        }

        public Proxy(string path, int port)
        {
            this.path = path;
            this.port = port;
        }

        public string Url
        {
            get { return string.Format("http://{0}:{1}", Host, this.port.ToString(CultureInfo.InvariantCulture)); }
        }

        // Start the server and connect to proxy
        public void Start()
        {
             this.serverProcess = new Process
            {
                StartInfo = { FileName = this.path }
            };
            if (this.port != 0)
            {
                this.serverProcess.StartInfo.Arguments = string.Format("--port={0}", this.port);
            }

          try
          {
              this.serverProcess.Start();
              int count = 0;
              while (!this.IsListening())
              {
                  Thread.Sleep(1000);
                  count++;
                  if (count == 30)
                  {
                      throw new Exception("Can not connect to specified Proxy");
                  }
              }
          }
          catch
          {
              this.serverProcess.Dispose();
              this.serverProcess = null;
              throw;
          }
        }

        // Stop the currently running proxy
        public void Stop()
        {
            if (this.serverProcess != null && !this.serverProcess.HasExited)
            {
                this.serverProcess.CloseMainWindow();
                this.serverProcess.Dispose();
                this.serverProcess.Close();
            }
        }

        public Client CreateProxy()
        {
            return new Client(this.Url);
        }

        private static WebResponse MakeRequest(string url, string method, string reference = null)
        {
            Console.WriteLine(url);
            var request = (HttpWebRequest)WebRequest.Create(url);
            request.Method = method;
            if (reference != null)
            {
                byte[] requestBytes = Encoding.UTF8.GetBytes(reference);
                using (var requestStream = request.GetRequestStream())
                {
                    requestStream.Write(requestBytes, 0, requestBytes.Length);
                    requestStream.Close();
                }

                request.ContentType = "application/x-www-form-urlencoded";
            }
            else
            {
                request.ContentLength = 0;
            }

            return request.GetResponse();
        }

        private bool IsListening()
        {
            try
            {
                var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
                socket.Connect(Host, this.port);
                socket.Close();
                return true;
            }
            catch (Exception)
            {
                return false;
            }
        }
    }


The client class looks like this:

public class Client
    {
        private static string proxy;
        private readonly string url;
        private readonly short port;
        private readonly string baseUrlProxy;

        public Client(string url)
        {
            if (string.IsNullOrEmpty(url))
            {
                throw new ArgumentException("url not supplied", "url");
            }

            this.url = url;
            this.baseUrlProxy = string.Format("{0}/proxy", this.url);
            using (var response = MakeRequest(this.baseUrlProxy, "POST"))
            {
                var responseStream = response.GetResponseStream();
                if (responseStream == null)
                {
                    throw new Exception("No response from proxy");
                }

                using (var responseStreamReader = new StreamReader(responseStream))
                {
                    var jsonReader = new JsonTextReader(responseStreamReader);
                    var token = JToken.ReadFrom(jsonReader);
                    var portToken = token.SelectToken("port");
                    if (portToken == null)
                    {
                        throw new Exception("No port number returned from proxy");
                    }

                    this.port = (short)portToken;
                }
            }

            var parts = url.Split(':');
            proxy = parts[1].TrimStart('/') + ":" + this.port;
        }

        public string SeleniumProxy
        {
            get { return proxy; }
        }

        public Har GetHar()
        {
            using (var response = MakeRequest(string.Format("{0}/{1}/har", this.baseUrlProxy, this.port), "GET"))
            {
                using (var responseStream = response.GetResponseStream())
                {
                    if (responseStream == null)
                    {
                        return null;
                    }

                    using (var responseStreamReader = new StreamReader(responseStream))
                    {
                        var json = responseStreamReader.ReadToEnd();
                        return JsonConvert.DeserializeObject<Har>(json);
                    }
                }
            }
        }

        public string GetHarString()
        {
            using (var response = MakeRequest(string.Format("{0}/{1}/har", this.baseUrlProxy, this.port), "GET"))
            {
                using (var responseStream = response.GetResponseStream())
                {
                    if (responseStream == null)
                    {
                        return null;
                    }

                    using (var responseStreamReader = new StreamReader(responseStream))
                    {
                        var json = responseStreamReader.ReadToEnd();
                        return json;
                    }
                }
            }
        }

        public void NewHar(string reference = null)
        {
            MakeRequest(string.Format("{0}/{1}/har", this.baseUrlProxy, this.port), "PUT", reference);
        }

        public void NewPage(string reference)
        {
            MakeRequest(string.Format("{0}/{1}/har/pageRef", this.baseUrlProxy, this.port), "PUT", reference);
        }

        public void RemapHost(string host, string ipaddress)
        {
            MakeJsonRequest(string.Format("{0}/{1}/hosts", this.baseUrlProxy, this.port), "POST", "{\"" + host + "\":\"" + ipaddress + "\"}");
        }

        /// <summary>
        /// shuts down the proxy and closes the port
        /// </summary>
        public void Close()
        {
            MakeRequest(string.Format("{0}/{1}", this.baseUrlProxy, this.port), "DELETE");
        }

        private static WebResponse MakeRequest(string url, string method, string reference = null)
        {
            var request = (HttpWebRequest)WebRequest.Create(url);
            request.Method = method;
            if (reference != null)
            {
                byte[] requestBytes = Encoding.UTF8.GetBytes(reference);
                using (var requestStream = request.GetRequestStream())
                {
                    requestStream.Write(requestBytes, 0, requestBytes.Length);
                    requestStream.Close();
                }

                request.ContentType = "application/x-www-form-urlencoded";
            }
            else
            {
                request.ContentLength = 0;
            }

            return request.GetResponse();
        }

        private static WebResponse MakeJsonRequest(string url, string method, string payload)
        {
            var request = (HttpWebRequest)WebRequest.Create(url);
            request.Method = method;

            if (payload != null)
            {
                request.ContentType = "text/json";
                request.ContentLength = payload.Length;
                using (var streamWriter = new StreamWriter(request.GetRequestStream()))
                {
                    streamWriter.Write(payload);
                    streamWriter.Flush();
                    streamWriter.Close();
                }
            }
            else
            {
                request.ContentLength = 0;
            }

            return request.GetResponse();
        }
    }


Elsewhere in my code I'm simply getting the har data from the client for my Selenium test (after the test has ended) and saving this in a .har file.


Giuseppe D'Amico

unread,
Jun 18, 2019, 5:45:25 AM6/18/19
to BrowserMob Proxy
I have manged to retrieve headers modifying the Client code in the following way:

using AutomatedTester.BrowserMob.HAR;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.IO;
using System.Net;
using System.Text;
using System.Web;
using System.Web.UI.WebControls.WebParts;

namespace AutomatedTester.BrowserMob
{
    public class Client
    {
        private readonly string _url;
        private readonly Int16 _port;
        private readonly string _proxy;
        private readonly string _baseUrlProxy;
        private bool _captureHeaders = false;

        public Client(string url)
        {
            if (String.IsNullOrEmpty(url))
                throw new ArgumentException("url not supplied", "url");

            _url = url;
            _baseUrlProxy = String.Format("{0}/proxy", _url);
            using (var response = MakeRequest(_baseUrlProxy, "POST"))
            {
                var responseStream = response.GetResponseStream();
                if (responseStream == null)
                    throw new Exception("No response from proxy");

                using (var responseStreamReader = new StreamReader(responseStream))
                {
                    var jsonReader = new JsonTextReader(responseStreamReader);
                    var token = JToken.ReadFrom(jsonReader);
                    var portToken = token.SelectToken("port");
                    if (portToken == null)
                        throw new Exception("No port number returned from proxy");

                    _port = (Int16)portToken;
                }
            }

            var parts = url.Split(':');
            _proxy = parts[1].TrimStart('/') + ":" + _port;
        }

        public int GetPort()
        {
            return _port;
        }

        public void SetCaptureHeaders(bool captureHeaders = true)
        {
            _captureHeaders = captureHeaders;
        }

        public void NewHar(string reference = null)
        {
            string parameters = "";
            var request = $"{_baseUrlProxy}/{_port}/har";
            if (_captureHeaders)
            {
                parameters = parameters + "captureHeaders=true";
            }
            MakeRequest($"{request}?{parameters}", "PUT", reference);
        public void NewPage(string reference)
        {
            MakeRequest(String.Format("{0}/{1}/har/pageRef", _baseUrlProxy, _port), "PUT", reference);
        }

        public HarResult GetHar()
        {
            var response = MakeRequest(String.Format("{0}/{1}/har", _baseUrlProxy, _port), "GET");
            using (var responseStream = response.GetResponseStream())
            {
                if (responseStream == null)
                    return null;

                using (var responseStreamReader = new StreamReader(responseStream))
                {
                    var json = responseStreamReader.ReadToEnd();
                    return JsonConvert.DeserializeObject<HarResult>(json);
                }
            }
        }

        public void SetLimits(LimitOptions options)
        {
            if (options == null)
                throw new ArgumentNullException("options", "LimitOptions must be supplied");

            MakeRequest(String.Format("{0}/{1}/limit", _baseUrlProxy, _port), "PUT", options.ToFormData());
        }

        public string SeleniumProxy
        {
            get { return _proxy; }
        }

        public void WhiteList(string regexp, int statusCode)
        {
            string data = FormatBlackOrWhiteListFormData(regexp, statusCode);
            MakeRequest(String.Format("{0}/{1}/whitelist", _baseUrlProxy, _port), "PUT", data);
        }

        public void Blacklist(string regexp, int statusCode)
        {
            string data = FormatBlackOrWhiteListFormData(regexp, statusCode);
            MakeRequest(String.Format("{0}/{1}/blacklist", _baseUrlProxy, _port), "PUT", data);
        }

        public void RemapHost(string host, string ipAddress)
        {
            MakeJsonRequest(String.Format("{0}/{1}/hosts", _baseUrlProxy, _port), "POST", "{\"" + host + "\":\"" + ipAddress + "\"}");
        }

        private static string FormatBlackOrWhiteListFormData(string regexp, int statusCode)
        {
            return String.Format("regex={0}&status={1}", HttpUtility.UrlEncode(regexp), statusCode);
        }

        /// <summary>
        /// shuts down the proxy and closes the port
        /// </summary>
        public void Close()
        {
            MakeRequest(String.Format("{0}/{1}", _baseUrlProxy, _port), "DELETE");
        }

    }
}

Then I start the proxy this way:

_client = MobProxyServer.GetServer().CreateProxy();
_client.SetCaptureHeaders();
_client.NewHar("xxx");

Where MobProxyServer is just a helper class responsible to start the server if not already started and create a new proxies.

This might help someone else facing the same issue.

Giuseppe D'Amico

unread,
Jun 18, 2019, 5:47:43 AM6/18/19
to BrowserMob Proxy
I have mainly added a method to enable the headers capture and modified the NewHar method in the following way:
Reply all
Reply to author
Forward
0 new messages