NED download

[Gina]

Hello,

I am trying to get the NED data set for the Western US. I've passed in the '1/3 arc-second' data set and the bbox as specified in the documentation: https://ulmo.readthedocs.io/en/latest/api.html#module-ulmo.usgs.ned

import ulmo

ulmo.usgs.eros.get_raster('1/3 arc-second', [-131.718560, 30.263258, -98.551469, 49.660747])

However, I am getting this error when running the above code:

  In [7]: ulmo.usgs.eros.get_raster('1/3 arc-second', [-131.718560, 30.263258, -98.551469, 49.660747])

calling http://nimbus.cr.usgs.gov/index_service/Index_Service_JSON2.asmx/return_Download_Options with payload {'ProductIDs': '1/3 arc-second', 'callback': ''}

---------------------------------------------------------------------------

gaierror                                  Traceback (most recent call last)

~\Anaconda3\lib\site-packages\urllib3\connection.py in _new_conn(self)

    140             conn = connection.create_connection(

--> 141                 (self.host, self.port), self.timeout, **extra_kw)

    142

~\Anaconda3\lib\site-packages\urllib3\util\connection.py in create_connection(address, timeout, source_address, socket_options)

     59

---> 60     for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):

     61         af, socktype, proto, canonname, sa = res

~\Anaconda3\lib\socket.py in getaddrinfo(host, port, family, type, proto, flags)

    744     addrlist = []

--> 745     for res in _socket.getaddrinfo(host, port, family, type, proto, flags):

    746         af, socktype, proto, canonname, sa = res

gaierror: [Errno 11001] getaddrinfo failed

During handling of the above exception, another exception occurred:

NewConnectionError                        Traceback (most recent call last)

~\Anaconda3\lib\site-packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)

    600                                                   body=body, headers=headers,

--> 601                                                   chunked=chunked)

    602

~\Anaconda3\lib\site-packages\urllib3\connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)

    356         else:

--> 357             conn.request(method, url, **httplib_request_kw)

    358

~\Anaconda3\lib\http\client.py in request(self, method, url, body, headers, encode_chunked)

   1238         """Send a complete request to the server."""

-> 1239         self._send_request(method, url, body, headers, encode_chunked)

   1240

~\Anaconda3\lib\http\client.py in _send_request(self, method, url, body, headers, encode_chunked)

   1284             body = _encode(body, 'body')

-> 1285         self.endheaders(body, encode_chunked=encode_chunked)

   1286

~\Anaconda3\lib\http\client.py in endheaders(self, message_body, encode_chunked)

   1233             raise CannotSendHeader()

-> 1234         self._send_output(message_body, encode_chunked=encode_chunked)

   1235

~\Anaconda3\lib\http\client.py in _send_output(self, message_body, encode_chunked)

   1025         del self._buffer[:]

-> 1026         self.send(msg)

   1027

~\Anaconda3\lib\http\client.py in send(self, data)

    963             if self.auto_open:

--> 964                 self.connect()

    965             else:

~\Anaconda3\lib\site-packages\urllib3\connection.py in connect(self)

    165     def connect(self):

--> 166         conn = self._new_conn()

    167         self._prepare_conn(conn)

~\Anaconda3\lib\site-packages\urllib3\connection.py in _new_conn(self)

    149             raise NewConnectionError(

--> 150                 self, "Failed to establish a new connection: %s" % e)

    151

NewConnectionError: <urllib3.connection.HTTPConnection object at 0x000001E4C78A9A58>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed

During handling of the above exception, another exception occurred:

MaxRetryError                             Traceback (most recent call last)

~\Anaconda3\lib\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)

    439                     retries=self.max_retries,

--> 440                     timeout=timeout

    441                 )

~\Anaconda3\lib\site-packages\urllib3\connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)

    638             retries = retries.increment(method, url, error=e, _pool=self,

--> 639                                         _stacktrace=sys.exc_info()[2])

    640             retries.sleep()

~\Anaconda3\lib\site-packages\urllib3\util\retry.py in increment(self, method, url, response, error, _pool, _stacktrace)

    387         if new_retry.is_exhausted():

--> 388             raise MaxRetryError(_pool, url, error or ResponseError(cause))

    389

MaxRetryError: HTTPConnectionPool(host='igskmncnvs641.cr.usgs.gov', port=80): Max retries exceeded with url: /custerr/en-US/500.aspx?aspxerrorpath=/index_service/Index_Service_JSON2.asmx/return_Download_Options (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000001E4C78A9A58>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed',))

During handling of the above exception, another exception occurred:

ConnectionError                           Traceback (most recent call last)

<ipython-input-7-3da65d062610> in <module>()

----> 1 ulmo.usgs.eros.get_raster('1/3 arc-second', [-131.718560, 30.263258, -98.551469, 49.660747])

~\Anaconda3\lib\site-packages\ulmo\usgs\eros\core.py in get_raster(product_key, bbox, fmt, path, check_modified, mosaic)

    194         metadata as a FeatureCollection. local url of downloaded data is in feature['properties']['file']

    195     """

--> 196     raster_tiles = _download_tiles(get_raster_availability(product_key, bbox, fmt),

    197         path, check_modified)

    198

~\Anaconda3\lib\site-packages\ulmo\usgs\eros\core.py in get_raster_availability(product_key, bbox, fmt)

    148

    149     xmin, ymin, xmax, ymax = [float(n) for n in bbox]

--> 150     layer, fmt = _layer_id(product_key, fmt)

    151

    152     url = EROS_VALIDATION_URL % (ymax, ymin, xmin, xmax, layer)

~\Anaconda3\lib\site-packages\ulmo\usgs\eros\core.py in _layer_id(product_key, fmt)

    278

    279 def _layer_id(product_key, fmt=None):

--> 280     available_formats = get_available_formats(product_key)

    281     if not available_formats.empty:

    282         available_formats = available_formats['outputformat'][0].lower()

~\Anaconda3\lib\site-packages\ulmo\usgs\eros\core.py in get_available_formats(product_key, as_dataframe)

    124     url = EROS_INVENTORY_URL + '/return_Download_Options'

    125     payload = {'ProductIDs': product_key}

--> 126     return _call_service(url, payload, as_dataframe)

    127

    128

~\Anaconda3\lib\site-packages\ulmo\usgs\eros\core.py in _call_service(url, payload, as_dataframe)

    243     payload['callback'] = ''

    244     print('calling %s with payload %s' % (url, repr(payload)))

--> 245     r = requests.get(url, params=payload)

    246     if as_dataframe:

    247         df = pd.DataFrame(r.json()['items'])

~\Anaconda3\lib\site-packages\requests\api.py in get(url, params, **kwargs)

     70

     71     kwargs.setdefault('allow_redirects', True)

---> 72     return request('get', url, params=params, **kwargs)

     73

     74

~\Anaconda3\lib\site-packages\requests\api.py in request(method, url, **kwargs)

     56     # cases, and look like a memory leak in others.

     57     with sessions.Session() as session:

---> 58         return session.request(method=method, url=url, **kwargs)

     59

     60

~\Anaconda3\lib\site-packages\requests\sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)

    506         }

    507         send_kwargs.update(settings)

--> 508         resp = self.send(prep, **send_kwargs)

    509

    510         return resp

~\Anaconda3\lib\site-packages\requests\sessions.py in send(self, request, **kwargs)

    638

    639         # Resolve redirects if allowed.

--> 640         history = [resp for resp in gen] if allow_redirects else []

    641

    642         # Shuffle things around if there's history.

~\Anaconda3\lib\site-packages\requests\sessions.py in <listcomp>(.0)

    638

    639         # Resolve redirects if allowed.

--> 640         history = [resp for resp in gen] if allow_redirects else []

    641

    642         # Shuffle things around if there's history.

~\Anaconda3\lib\site-packages\requests\sessions.py in resolve_redirects(self, resp, req, stream, timeout, verify, cert, proxies, yield_requests, **adapter_kwargs)

    216                     proxies=proxies,

    217                     allow_redirects=False,

--> 218                     **adapter_kwargs

    219                 )

    220

~\Anaconda3\lib\site-packages\requests\sessions.py in send(self, request, **kwargs)

    616

    617         # Send the request

--> 618         r = adapter.send(request, **kwargs)

    619

    620         # Total elapsed time of the request (approximately)

~\Anaconda3\lib\site-packages\requests\adapters.py in send(self, request, stream, timeout, verify, cert, proxies)

    506                 raise SSLError(e, request=request)

    507

--> 508             raise ConnectionError(e, request=request)

    509

    510         except ClosedPoolError as e:

ConnectionError: HTTPConnectionPool(host='igskmncnvs641.cr.usgs.gov', port=80): Max retries exceeded with url: /custerr/en-US/500.aspx?aspxerrorpath=/index_service/Index_Service_JSON2.asmx/return_Download_Options (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000001E4C78A9A58>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed',))

Any idea what might be the issue?

Thanks,

Gina

[Gina]

Just as a followup -- I've sort of resolved my issue. I passed in a different set of coordinates (roughly the same area) and now it is working. Here is the new call:

In [23]: ulmo.usgs.ned.get_raster('1/3 arc-second', bbox=[-129.226343, 31.042818, -99.589304, 49.710378], path="C:\\Use

    ...: rs\\ginal\\Documents\\EarthLab\\NED_ulmo")

retrieving raster availability from https://www.sciencebase.gov/catalog/items?parentId=4f70aa9fe4b058caae3f8de5&filter=tags%3DIMG&max=1000&fields=webLinks%2Cspatial%2Ctitle&format=json&filter=spatialQuery%3D%7Bwkt%3A%22POLYGON+%28%28-129.226343+49.710378%2C-129.226343+31.042818%2C-99.589304+31.042818%2C-99.589304+49.710378%2C-129.226343+49.710378%29%29%22%2Crelation%3A%22intersects%22%7D

... downloading tile 1 of 1 from https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/IMG/USGS_NED_13_n32w107_IMG.zip

... ... zipfile saved at C:\Users\ginal\Documents\EarthLab\NED_ulmo\4f70aa9fe4b058caae3f8de5\zip\USGS_NED_13_n32w107_IMG.zip

... ... .img format raster saved at C:\Users\ginal\Documents\EarthLab\NED_ulmo\4f70aa9fe4b058caae3f8de5\USGS_NED_13_n32w107_IMG.img

... downloading tile 1 of 1 from https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/IMG/USGS_NED_13_n48w100_IMG.zip

[Dharhas Pothina]

In the first you are using ulmo.usgs.eros.get_raster

and in the second you are using ulmo.usgs.ned.get_raster

Those two commands hit two different USGS web services EROS and NED. I haven't checked recently but I'm pretty sure the EROS web site no longer works...

- dharhas

--
You received this message because you are subscribed to the Google Groups "ulmo" group.
To unsubscribe from this group and stop receiving emails from it, send an email to ulmo+uns...@googlegroups.com.
To post to this group, send email to ul...@googlegroups.com.
Visit this group at https://groups.google.com/group/ulmo.
For more options, visit https://groups.google.com/d/optout.

[Emilio Mayorga]

On Wed, Sep 5, 2018 at 12:27 PM Dharhas Pothina <dha...@gmail.com> wrote:

In the first you are using ulmo.usgs.eros.get_raster

and in the second you are using ulmo.usgs.ned.get_raster

Those two commands hit two different USGS web services EROS and NED. I haven't checked recently but I'm pretty sure the EROS web site no longer works...

I think I ran into this EROS problem a couple of months ago, too, possibly (my memory is sketchy) both with ulmo and USGS-advertised web service links. This EROS service url (part of the original error message reported) leads to an inaccessible site:

http://nimbus.cr.usgs.gov/index_service/Index_Service_JSON2.asmx

Cheers,

-Emilio

[gili...@colorado.edu]

Hi,

Thank you both, you are right. It was a simple typo. However, I've also since run into another NED issue:

I ran the get_raster() function on my bounding box and let it run for a few hours (it's a large area -- essentially the western US). I piped the output to a log file. Almost all NED tiles had finished downloading (404 out of around 417 or so tiles) when this error was received:

... ... zipfile saved at /home/ubuntu/NED/4f70aa9fe4b058caae3f8de5/zip/USGS_NED_13_n39w114_IMG.zip

... ... .img format raster saved at /home/ubuntu/NED/4f70aa9fe4b058caae3f8de5/USGS_NED_13_n39w114_IMG.img

... downloading tile 1 of 1 from https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/IMG/USGS_NED_13_n41w117_IMG.zip

... ... zipfile saved at /home/ubuntu/NED/4f70aa9fe4b058caae3f8de5/zip/USGS_NED_13_n41w117_IMG.zip

... ... .img format raster saved at /home/ubuntu/NED/4f70aa9fe4b058caae3f8de5/USGS_NED_13_n41w117_IMG.img

... downloading tile 1 of 1 from https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/IMG/USGS_NED_13_n37w114_IMG.zip

... ... zipfile saved at /home/ubuntu/NED/4f70aa9fe4b058caae3f8de5/zip/USGS_NED_13_n37w114_IMG.zip

... ... .img format raster saved at /home/ubuntu/NED/4f70aa9fe4b058caae3f8de5/USGS_NED_13_n37w114_IMG.img

... downloading tile 1 of 1 from https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/IMG/USGS_NED_13_n36w106_IMG.zip

... ... zipfile saved at /home/ubuntu/NED/4f70aa9fe4b058caae3f8de5/zip/USGS_NED_13_n36w106_IMG.zip

... ... .img format raster saved at /home/ubuntu/NED/4f70aa9fe4b058caae3f8de5/USGS_NED_13_n36w106_IMG.img

... downloading tile 1 of 1 from https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/IMG/USGS_NED_13_n33w110_IMG.zip

... ... zipfile saved at /home/ubuntu/NED/4f70aa9fe4b058caae3f8de5/zip/USGS_NED_13_n33w110_IMG.zip

... ... .img format raster saved at /home/ubuntu/NED/4f70aa9fe4b058caae3f8de5/US

GS_NED_13_n33w110_IMG.img

... downloading tile 1 of 1 from https://prd-tnm.s3.amazonaws.com/StagedProducts/Elevation/13/IMG/USGS_NED_13_n35w100_IMG.zip

... ... zipfile saved at /home/ubuntu/NED/4f70aa9fe4b058caae3f8de5/zip/USGS_NED_13_n35w100_IMG.zip

Traceback (most recent call last):

  File "ned_download.py", line 2, in <module>

    ulmo.usgs.ned.get_raster('1/3 arc-second', bbox=[-129.226343, 31.042818, -99.589304, 49.710378], path="/home/ubuntu/NED/")

  File "/home/ubuntu/.local/lib/python2.7/site-packages/ulmo/usgs/ned/core.py", line 141, in get_raster

    check_modified=check_modified)

  File "/home/ubuntu/.local/lib/python2.7/site-packages/ulmo/usgs/ned/core.py", line 213, in _download_tiles

    tile['properties']['file'] = util.download_tiles(layer_path, metadata['download url'], metadata['format'], check_modified)[0]

  File "/home/ubuntu/.local/lib/python2.7/site-packages/ulmo/util/raster.py", line 55, in download_tiles

    tile_path = extract_from_zip(zip_path, tile_path, tile_fmt)

  File "/home/ubuntu/.local/lib/python2.7/site-packages/ulmo/util/raster.py", line 64, in extract_from_zip

    fname = [x for x in z.namelist() if tile_fmt in x[-4:]][0]

IndexError: list index out of range

Any idea what could have gone wrong?