httplib.BadStatusLine in Python/Selenium/PhantomJS

[Josh MacDonald]

This error has been under my skin for a few hours now. I decided to code up a separate project just to see if I can replicate it and I can, but ONLY on my server. This works on my Mac.

Mac: OSX El Capitan 10.11.6

Server: CentOS 7.2.1511

Both have PhantomJS version: 2.1.1

Python Mac: Python 2.7.11

Python Server: 2.7.5

Both have Selenium version: 2.53.0

Identical code ran on both:

from selenium import webdriver



from selenium.webdriver.common.by import By

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.support import expected_conditions as EC

from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

from selenium.common.exceptions import NoSuchElementException

import time




dcap = dict(DesiredCapabilities.PHANTOMJS)

dcap["phantomjs.page.settings.userAgent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36"

dcap["phantomjs.page.customHeaders.accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"

dcap["phantomjs.page.customHeaders.Accept-Language"] = "en-US,en;q=0.8"

dcap["phantomjs.page.customHeaders.connection"] = "keep-alive"




driver = webdriver.PhantomJS(desired_capabilities=dcap)

driver.set_window_size(1120, 700)

driver.get("https://www.instagram.com/espn/")




while True:

    print len(driver.find_elements_by_css_selector("a[href*='/p/']"))

    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

    try:

        loadMore = driver.find_element_by_link_text("Load more")

        loadMore.click()

    except NoSuchElementException:

        print "No such"

        driver.save_screenshot('none.png')

Mac output:

12

24

No such

24

No such

36

No such

48

No such

48

No such

60

No such

72

No such

84

# This goes until I end it

Server output:

12



24

No such

Traceback (most recent call last):

  File "junk.py", line 27, in <module>

    driver.save_screenshot('none.png')

  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 790, in get_screenshot_as_file

    png = self.get_screenshot_as_png()

  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 809, in get_screenshot_as_png

    return base64.b64decode(self.get_screenshot_as_base64().encode('ascii'))

  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 819, in get_screenshot_as_base64

    return self.execute(Command.SCREENSHOT)['value']

  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 231, in execute

    response = self.command_executor.execute(driver_command, params)

  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/remote_connection.py", line 395, in execute

    return self._request(command_info[0], url, body=data)

  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/remote_connection.py", line 463, in _request

    resp = opener.open(request, timeout=self._timeout)

  File "/usr/lib64/python2.7/urllib2.py", line 431, in open

    response = self._open(req, data)

  File "/usr/lib64/python2.7/urllib2.py", line 449, in _open

    '_open', req)

  File "/usr/lib64/python2.7/urllib2.py", line 409, in _call_chain

    result = func(*args)

  File "/usr/lib64/python2.7/urllib2.py", line 1244, in http_open

    return self.do_open(httplib.HTTPConnection, req)

  File "/usr/lib64/python2.7/urllib2.py", line 1217, in do_open

    r = h.getresponse(buffering=True)

  File "/usr/lib64/python2.7/httplib.py", line 1089, in getresponse

    response.begin()

  File "/usr/lib64/python2.7/httplib.py", line 444, in begin

    version, status, reason = self._read_status()

  File "/usr/lib64/python2.7/httplib.py", line 408, in _read_status

    raise BadStatusLine(line)

httplib.BadStatusLine: ''

Server output after removing the screenshot line:

12



24

No such

24

Traceback (most recent call last):

  File "junk.py", line 23, in <module>

    loadMore = driver.find_element_by_link_text("Load more")

  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 314, in find_element_by_link_text

    return self.find_element(by=By.LINK_TEXT, value=link_text)

  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 744, in find_element

    {'using': by, 'value': value})['value']

  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 231, in execute

    response = self.command_executor.execute(driver_command, params)

  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/remote_connection.py", line 395, in execute

    return self._request(command_info[0], url, body=data)

  File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/remote_connection.py", line 463, in _request

    resp = opener.open(request, timeout=self._timeout)

  File "/usr/lib64/python2.7/urllib2.py", line 431, in open

    response = self._open(req, data)

  File "/usr/lib64/python2.7/urllib2.py", line 449, in _open

    '_open', req)

  File "/usr/lib64/python2.7/urllib2.py", line 409, in _call_chain

    result = func(*args)

  File "/usr/lib64/python2.7/urllib2.py", line 1244, in http_open

    return self.do_open(httplib.HTTPConnection, req)

  File "/usr/lib64/python2.7/urllib2.py", line 1217, in do_open

    r = h.getresponse(buffering=True)

  File "/usr/lib64/python2.7/httplib.py", line 1089, in getresponse

    response.begin()

  File "/usr/lib64/python2.7/httplib.py", line 444, in begin

    version, status, reason = self._read_status()

  File "/usr/lib64/python2.7/httplib.py", line 408, in _read_status

    raise BadStatusLine(line)

httplib.BadStatusLine: ''