A lot of these posts helped me; so I am giving back. Here are some classes to help other integrate BrowserMob Proxy into Selenium

113 views
Skip to first unread message

Jeff White

unread,
Mar 20, 2019, 10:10:26 AM3/20/19
to BrowserMob Proxy


package com.browserproxy;
//CHECKSTYLE:OFF checkstyle:magicnumber

import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import net.lightbody.bmp.BrowserMobProxy;
import net.lightbody.bmp.BrowserMobProxyServer;
import net.lightbody.bmp.client.ClientUtil;
import net.lightbody.bmp.core.har.Har;
import net.lightbody.bmp.proxy.CaptureType;
import org.junit.Assert;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.remote.CapabilityType;
import java.io.File;
import java.io.IOException;
import java.net.*;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;

/**
* Browser Mob based class
*
*
*/

@Slf4j
public class BProxy {

@Getter
private static BrowserMobProxy proxy = null;
private int proxyPort = 9876;

/***********************************************************
* public BrowserMobProxy getProxyServer()
* Sets up and starts the Browser Mob Proxy
*
*
* @return Proxy instance
***********************************************************/
public void getProxyServer() throws UnknownHostException {
getProxyServer(CaptureType.REQUEST_CONTENT,
CaptureType.REQUEST_HEADERS,
CaptureType.RESPONSE_CONTENT,
CaptureType.RESPONSE_HEADERS);
}

/**
*
* @param captureTypes Comma Separated list
* @throws UnknownHostException
*
* - Sets up and starts the Browser Mob Proxy
* * - Allows the ability to set capture types for the HAR capture
* *
* * EX:
* * getProxyServer(CaptureType.REQUEST_CONTENT,
* * CaptureType.REQUEST_HEADERS,
* * CaptureType.RESPONSE_CONTENT,
* * CaptureType.RESPONSE_HEADERS);
*
*/
public void getProxyServer(CaptureType... captureTypes) throws UnknownHostException {
getProxyServer(proxyPort, captureTypes);
}

/**
* public void getProxyServer(int port, CaptureType... captureTypes)
* @param port - Port to start Proxy server
* @param captureTypes - Comma separated list of capture Types
* @throws UnknownHostException
*
* - Ability to set the port
* - Sets up and starts the Browser Mob Proxy
* - Allows the ability to set capture types for the HAR capture
*
*/

public void getProxyServer(int port, CaptureType... captureTypes) throws UnknownHostException {
if(proxy == null) {
proxy = new BrowserMobProxyServer();
proxy.setTrustAllServers(true);
InetAddress connectableAddress = ClientUtil.getConnectableAddress(); // Default behaviour
log.info("Connectable Address=" + connectableAddress);
// above line is needed for application with invalid certificates
log.info("Local Host Address=" + InetAddress.getLocalHost());
proxy.start(port, InetAddress.getByName("localhost"));
log.info("Local Host Address=" + proxy.getClientBindAddress());
log.info("Starting Browser Mob Proxy");
proxy.setHarCaptureTypes(CaptureType.getAllContentCaptureTypes());
proxy.enableHarCaptureTypes(captureTypes);
}
}

/***********************************************************
* private InetSocketAddress getNetProxy()
* @return address of proxy
*
* Returns the address of the network proxy being used
***********************************************************/
protected InetSocketAddress getNetProxy() {
System.setProperty("java.net.useSystemProxies", "true");
List<java.net.Proxy> l = null;
try {
l = ProxySelector.getDefault().select(
new URI("http://www.google.com/"));
} catch (URISyntaxException e1) {
e1.printStackTrace();
}

InetSocketAddress addr = null;
for (Iterator<java.net.Proxy> iter = l.iterator(); iter.hasNext(); ) {
java.net.Proxy myProxy = iter.next();
System.out.println("proxy hostname : " + myProxy.type());
addr = (InetSocketAddress) myProxy.address();

if (addr == null) {
log.info("No Proxy");
} else {
log.info("proxy hostname : " + addr.getHostName());
log.info("proxy port : " + addr.getPort());
}
}

return addr;
}

/***********************************************************
* public Proxy setSeleniumProxy(BrowserMobProxy proxyServer)
* @param proxyServer
* @return seleniumProxy
*
* Sets up the Selenium Proxy
***********************************************************/
public Proxy getSeleniumProxy(BrowserMobProxy proxyServer) {
Proxy seleniumProxy = ClientUtil.createSeleniumProxy(proxyServer);
try {
String hostIp = "localhost";
seleniumProxy.setHttpProxy(hostIp + ":" + proxyServer.getPort());
seleniumProxy.setSslProxy(hostIp + ":" + proxyServer.getPort());
} catch (Exception e) {
e.printStackTrace();
Assert.fail("invalid Host Address");
}
return seleniumProxy;
}

/***********************************************************
* private ChromeOptions getChromeOptions()
* Sets the Chrome Options for Browser Mob Proxy
* @return options
*
* This is for any apps using this outside of
* the selenium framework.
***********************************************************/
protected ChromeOptions getChromeOptions() {
ChromeOptions options = new ChromeOptions();
options.setCapability(CapabilityType.ACCEPT_SSL_CERTS, true);
options.setCapability(CapabilityType.SUPPORTS_JAVASCRIPT, true);
options.setCapability(CapabilityType.PROXY, getSeleniumProxy(proxy));

return options;
}

/***********************************************************
* public void quit()
* Gives the ability to close (quit) the Browser Mob Proxy
***********************************************************/
public void quit() {
proxy.stop();
}

/***********************************************************
* private void writeHar(Har har, String filePath)
* @param har
* @param filePath
* Handles writing the Har file based upon path
*
***********************************************************/
protected void writeHar(Har har, String filePath) {
try {
proxy.getHar().writeTo(new File(filePath));
} catch (IOException e) {
e.printStackTrace();
}
}

/**
* protected void writeHar()
*/
protected void writeHar(){
writeHar(getHar(), "target/test.har");
}

/**
* public void blacklistRequests(String blackListCollection, int RespCode)
* @param blackListCollection
* @param RespCode
*
* Allows user to add items to blacklist from results
* Ex: blacklistRequests("https?://.*\\.google-analytics\\.com/.*", 410);
* 410 is essentially "Gone"
*/
public void blacklistRequests(String blackListCollection, int RespCode){
proxy.blacklistRequests(blackListCollection, RespCode);
}

/**
* public void whitelistRequests(Collection<String> collection, int StatusCode)
* @param collection
* @param StatusCode
*
* Ex: whitelistRequests("https?://*.*.yoursite.com/.*. https://*.*.someOtherYourSite.*".split(","), 200);
*
* Allows user to add URL patterns to allow
*/
public void whitelistRequests(Collection<String> collection, int StatusCode){
proxy.whitelistRequests(collection, StatusCode);
}

/***********************************************************
* public void newHar(String name)
* @param name
*
* Create a new HAR with the label specified by the parameter 'name'
***********************************************************/
public void newHar(String name) {
proxy.newHar(name);
}

/***********************************************************
* public void newHar()
*
* Create new Har
***********************************************************/
public void newHar() {
proxy.newHar();
}

/***********************************************************
* public void newPage()
*
* Creates a new page entry
***********************************************************/
public void newPage() {
proxy.newPage();
}

/***********************************************************
* public void newPage(String pageName)
* @param pageName
* Designates a new page in the HAR
***********************************************************/
public void newPage(String pageName) {
proxy.newPage(pageName);
}

/***********************************************************
* public Har getHar()
* @return
*
* Collect the performance data from the BrowserMob proxy server.
* Get the HAR data.
***********************************************************/
public Har getHar() {
return proxy.getHar();
}

/***********************************************************
* public void endHar()
*
* Ends the HAR file
***********************************************************/
public void endHar() {
proxy.endHar();
}




}


Bean Declaration:

############################################################################

package com.browserproxy;

import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Service;

@Service
public class BProxyConfig {

@Bean
public BProxy bProxy() {
return new BProxy();
}
}


############################################################################

Mar 18


The usage of those two classes:

//CHECKSTYLE:OFF checkstyle:magicnumber

import io.github.bonigarcia.wdm.ChromeDriverManager;
import lombok.extern.slf4j.Slf4j;
import net.lightbody.bmp.proxy.CaptureType;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;



@Slf4j
@RunWith(SpringRunner.class)
@SpringBootTest
public class BProxyIT {

@Autowired
    private  BProxy bProxy;

/***********************************************************
* public void bproxyTest()
*
* Test of BrowserMob Proxy
**********************************************************/

   @Test
public void bproxyTest() {

ChromeDriverManager.chromedriver().setup();
try {
bProxy.getProxyServer(CaptureType.REQUEST_HEADERS, CaptureType.RESPONSE_HEADERS);
} catch (UnknownHostException e) {
e.printStackTrace();
}

// Check for Proxy (Currently turned off on the NextGear network
InetSocketAddress addr = bProxy.getNetProxy();

if (addr != null) {
bProxy.getProxy().setChainedProxy(bProxy.getNetProxy());
}

ChromeOptions chromeOptions = bProxy.getChromeOptions();
WebDriver driver = new ChromeDriver(chromeOptions);
bProxy.newHar();
bProxy.newPage("Youtube Home Page");

List<String> allowUrlPatterns = new ArrayList<String>();
allowUrlPatterns.add("https?://.*(youtube.com)+.*");
bProxy.whitelistRequests(allowUrlPatterns, 200);
String blackListedItems = ".*(.cdn.*|.png|.jpg|.js)";
bProxy.blacklistRequests(blackListedItems, 410);
driver.get("http://www.youtube.com");
//driver.findElement(By.name("q")).sendKeys("Browser Mob Proxy");
bProxy.writeHar(bProxy.getHar(), "target/test.har");
log.info("Shutting down Browser Mob Proxy");

bProxy.quit();
driver.quit();
}
}

Jeff White

unread,
Mar 20, 2019, 10:11:30 AM3/20/19
to BrowserMob Proxy
And for looking at the HAR file routes:




import de.sstoehr.harreader.HarReader;
import de.sstoehr.harreader.HarReaderException;
import de.sstoehr.harreader.model.Har;
import de.sstoehr.harreader.model.HarEntry;
import lombok.extern.slf4j.Slf4j;
import java.io.File;
import java.util.List;

@Slf4j
public class HarCompare {

public void harCompare() throws HarReaderException {
HarReader harReader = new HarReader();
Har har = harReader.readFromFile(new File("target/test.har"));
List<HarEntry> entries = har.getLog().getEntries();
for (HarEntry entry: entries) {
String method = entry.getRequest().getMethod().name();
String url = entry.getRequest().getUrl();
System.out.println(url);
}
}

public static void main(String[] args) throws HarReaderException {
HarCompare hc = new HarCompare();
hc.harCompare();
}




}
Reply all
Reply to author
Forward
0 new messages